2024-11-14 06:15:59,237 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-14 06:15:59,257 main DEBUG Took 0.016452 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-14 06:15:59,258 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-14 06:15:59,258 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-14 06:15:59,260 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-14 06:15:59,262 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 06:15:59,281 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-14 06:15:59,305 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 06:15:59,307 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 06:15:59,308 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 06:15:59,309 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 06:15:59,310 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 06:15:59,310 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 06:15:59,312 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 06:15:59,314 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 06:15:59,315 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 06:15:59,316 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 06:15:59,318 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 06:15:59,319 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 06:15:59,320 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 06:15:59,320 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 06:15:59,321 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 06:15:59,321 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 06:15:59,322 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 06:15:59,323 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 06:15:59,323 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 06:15:59,324 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 06:15:59,325 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 06:15:59,325 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 06:15:59,326 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 06:15:59,327 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 06:15:59,327 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 06:15:59,328 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-14 06:15:59,331 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 06:15:59,333 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-14 06:15:59,335 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-14 06:15:59,336 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-14 06:15:59,338 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-14 06:15:59,339 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-14 06:15:59,350 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-14 06:15:59,354 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-14 06:15:59,357 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-14 06:15:59,358 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-14 06:15:59,359 main DEBUG createAppenders(={Console}) 2024-11-14 06:15:59,360 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-14 06:15:59,361 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-14 06:15:59,361 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-14 06:15:59,362 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-14 06:15:59,362 main DEBUG OutputStream closed 2024-11-14 06:15:59,363 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-14 06:15:59,363 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-14 06:15:59,363 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-14 06:15:59,540 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-14 06:15:59,543 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-14 06:15:59,545 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-14 06:15:59,547 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-14 06:15:59,548 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-14 06:15:59,548 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-14 06:15:59,549 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-14 06:15:59,549 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-14 06:15:59,549 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-14 06:15:59,550 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-14 06:15:59,551 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-14 06:15:59,551 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-14 06:15:59,552 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-14 06:15:59,552 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-14 06:15:59,553 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-14 06:15:59,553 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-14 06:15:59,553 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-14 06:15:59,555 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-14 06:15:59,558 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-14 06:15:59,558 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-14 06:15:59,558 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-14 06:15:59,560 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-14T06:15:59,976 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5ae4a4a-2751-c4f4-7a28-da4c7cf058aa 2024-11-14 06:15:59,980 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-14 06:15:59,981 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-14T06:15:59,992 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-14T06:16:00,022 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-14T06:16:00,047 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5ae4a4a-2751-c4f4-7a28-da4c7cf058aa/cluster_7988e05c-75f7-7e7d-81fe-ae6ab708e25a, deleteOnExit=true 2024-11-14T06:16:00,048 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-14T06:16:00,049 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5ae4a4a-2751-c4f4-7a28-da4c7cf058aa/test.cache.data in system properties and HBase conf 2024-11-14T06:16:00,050 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5ae4a4a-2751-c4f4-7a28-da4c7cf058aa/hadoop.tmp.dir in system properties and HBase conf 2024-11-14T06:16:00,052 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5ae4a4a-2751-c4f4-7a28-da4c7cf058aa/hadoop.log.dir in system properties and HBase conf 2024-11-14T06:16:00,053 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5ae4a4a-2751-c4f4-7a28-da4c7cf058aa/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-14T06:16:00,054 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5ae4a4a-2751-c4f4-7a28-da4c7cf058aa/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-14T06:16:00,054 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-14T06:16:00,174 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-14T06:16:00,300 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T06:16:00,306 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5ae4a4a-2751-c4f4-7a28-da4c7cf058aa/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-14T06:16:00,307 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5ae4a4a-2751-c4f4-7a28-da4c7cf058aa/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-14T06:16:00,307 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5ae4a4a-2751-c4f4-7a28-da4c7cf058aa/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-14T06:16:00,309 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5ae4a4a-2751-c4f4-7a28-da4c7cf058aa/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-14T06:16:00,310 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5ae4a4a-2751-c4f4-7a28-da4c7cf058aa/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-14T06:16:00,311 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5ae4a4a-2751-c4f4-7a28-da4c7cf058aa/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-14T06:16:00,312 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5ae4a4a-2751-c4f4-7a28-da4c7cf058aa/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-14T06:16:00,313 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5ae4a4a-2751-c4f4-7a28-da4c7cf058aa/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-14T06:16:00,314 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5ae4a4a-2751-c4f4-7a28-da4c7cf058aa/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-14T06:16:00,314 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5ae4a4a-2751-c4f4-7a28-da4c7cf058aa/nfs.dump.dir in system properties and HBase conf 2024-11-14T06:16:00,315 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5ae4a4a-2751-c4f4-7a28-da4c7cf058aa/java.io.tmpdir in system properties and HBase conf 2024-11-14T06:16:00,315 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5ae4a4a-2751-c4f4-7a28-da4c7cf058aa/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-14T06:16:00,315 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5ae4a4a-2751-c4f4-7a28-da4c7cf058aa/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-14T06:16:00,316 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5ae4a4a-2751-c4f4-7a28-da4c7cf058aa/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-14T06:16:01,396 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-14T06:16:01,485 INFO [Time-limited test {}] log.Log(170): Logging initialized @3142ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-14T06:16:01,588 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T06:16:01,677 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T06:16:01,705 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T06:16:01,705 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T06:16:01,707 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-14T06:16:01,737 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T06:16:01,740 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3f9fd429{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5ae4a4a-2751-c4f4-7a28-da4c7cf058aa/hadoop.log.dir/,AVAILABLE} 2024-11-14T06:16:01,741 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@40029c9a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T06:16:01,963 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2a5ffc6{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5ae4a4a-2751-c4f4-7a28-da4c7cf058aa/java.io.tmpdir/jetty-localhost-45981-hadoop-hdfs-3_4_1-tests_jar-_-any-17029702779175239539/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-14T06:16:01,978 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7f30a77e{HTTP/1.1, (http/1.1)}{localhost:45981} 2024-11-14T06:16:01,979 INFO [Time-limited test {}] server.Server(415): Started @3636ms 2024-11-14T06:16:02,657 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T06:16:02,665 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T06:16:02,666 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T06:16:02,666 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T06:16:02,666 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-14T06:16:02,667 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7d058786{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5ae4a4a-2751-c4f4-7a28-da4c7cf058aa/hadoop.log.dir/,AVAILABLE} 2024-11-14T06:16:02,667 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@50fd64ad{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T06:16:02,780 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@58e0e813{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5ae4a4a-2751-c4f4-7a28-da4c7cf058aa/java.io.tmpdir/jetty-localhost-39181-hadoop-hdfs-3_4_1-tests_jar-_-any-120544937247288192/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T06:16:02,780 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@716e8981{HTTP/1.1, (http/1.1)}{localhost:39181} 2024-11-14T06:16:02,781 INFO [Time-limited test {}] server.Server(415): Started @4438ms 2024-11-14T06:16:02,831 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-14T06:16:04,097 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5ae4a4a-2751-c4f4-7a28-da4c7cf058aa/cluster_7988e05c-75f7-7e7d-81fe-ae6ab708e25a/data/data1/current/BP-507516446-172.17.0.3-1731564960889/current, will proceed with Du for space computation calculation, 2024-11-14T06:16:04,097 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5ae4a4a-2751-c4f4-7a28-da4c7cf058aa/cluster_7988e05c-75f7-7e7d-81fe-ae6ab708e25a/data/data2/current/BP-507516446-172.17.0.3-1731564960889/current, will proceed with Du for space computation calculation, 2024-11-14T06:16:04,131 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-14T06:16:04,184 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x733d9428068e4da4 with lease ID 0xe8912deee189497a: Processing first storage report for DS-7f1dcb8e-c4d0-4e38-8a98-6bff453afa6a from datanode DatanodeRegistration(127.0.0.1:33461, datanodeUuid=93aa84dc-c036-4732-ac22-9c38de32e706, infoPort=44151, infoSecurePort=0, ipcPort=39501, storageInfo=lv=-57;cid=testClusterID;nsid=176468026;c=1731564960889) 2024-11-14T06:16:04,185 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x733d9428068e4da4 with lease ID 0xe8912deee189497a: from storage DS-7f1dcb8e-c4d0-4e38-8a98-6bff453afa6a node DatanodeRegistration(127.0.0.1:33461, datanodeUuid=93aa84dc-c036-4732-ac22-9c38de32e706, infoPort=44151, infoSecurePort=0, ipcPort=39501, storageInfo=lv=-57;cid=testClusterID;nsid=176468026;c=1731564960889), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-14T06:16:04,186 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x733d9428068e4da4 with lease ID 0xe8912deee189497a: Processing first storage report for DS-4dc2d572-9677-406a-9c56-2e5156dc5da2 from datanode DatanodeRegistration(127.0.0.1:33461, datanodeUuid=93aa84dc-c036-4732-ac22-9c38de32e706, infoPort=44151, infoSecurePort=0, ipcPort=39501, storageInfo=lv=-57;cid=testClusterID;nsid=176468026;c=1731564960889) 2024-11-14T06:16:04,186 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x733d9428068e4da4 with lease ID 0xe8912deee189497a: from storage DS-4dc2d572-9677-406a-9c56-2e5156dc5da2 node DatanodeRegistration(127.0.0.1:33461, datanodeUuid=93aa84dc-c036-4732-ac22-9c38de32e706, infoPort=44151, infoSecurePort=0, ipcPort=39501, storageInfo=lv=-57;cid=testClusterID;nsid=176468026;c=1731564960889), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T06:16:04,282 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5ae4a4a-2751-c4f4-7a28-da4c7cf058aa 2024-11-14T06:16:04,396 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5ae4a4a-2751-c4f4-7a28-da4c7cf058aa/cluster_7988e05c-75f7-7e7d-81fe-ae6ab708e25a/zookeeper_0, clientPort=61013, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5ae4a4a-2751-c4f4-7a28-da4c7cf058aa/cluster_7988e05c-75f7-7e7d-81fe-ae6ab708e25a/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5ae4a4a-2751-c4f4-7a28-da4c7cf058aa/cluster_7988e05c-75f7-7e7d-81fe-ae6ab708e25a/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-14T06:16:04,408 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=61013 2024-11-14T06:16:04,418 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T06:16:04,421 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T06:16:04,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741825_1001 (size=7) 2024-11-14T06:16:05,038 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02 with version=8 2024-11-14T06:16:05,038 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/hbase-staging 2024-11-14T06:16:05,123 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-14T06:16:05,321 INFO [Time-limited test {}] client.ConnectionUtils(128): master/ef93ef3a83c5:0 server-side Connection retries=45 2024-11-14T06:16:05,329 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T06:16:05,329 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T06:16:05,333 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T06:16:05,334 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T06:16:05,334 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T06:16:05,487 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-14T06:16:05,542 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-14T06:16:05,551 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-14T06:16:05,555 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T06:16:05,581 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 8765 (auto-detected) 2024-11-14T06:16:05,582 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-14T06:16:05,600 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:36045 2024-11-14T06:16:05,623 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36045 connecting to ZooKeeper ensemble=127.0.0.1:61013 2024-11-14T06:16:05,717 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:360450x0, quorum=127.0.0.1:61013, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T06:16:05,720 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36045-0x10137fc85b70000 connected 2024-11-14T06:16:05,814 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T06:16:05,816 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T06:16:05,824 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36045-0x10137fc85b70000, quorum=127.0.0.1:61013, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T06:16:05,828 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02, hbase.cluster.distributed=false 2024-11-14T06:16:05,851 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36045-0x10137fc85b70000, quorum=127.0.0.1:61013, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T06:16:05,855 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36045 2024-11-14T06:16:05,856 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36045 2024-11-14T06:16:05,858 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36045 2024-11-14T06:16:05,859 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36045 2024-11-14T06:16:05,859 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36045 2024-11-14T06:16:05,980 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/ef93ef3a83c5:0 server-side Connection retries=45 2024-11-14T06:16:05,981 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T06:16:05,982 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T06:16:05,982 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T06:16:05,982 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T06:16:05,982 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T06:16:05,985 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-14T06:16:05,988 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T06:16:05,994 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:42283 2024-11-14T06:16:05,998 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42283 connecting to ZooKeeper ensemble=127.0.0.1:61013 2024-11-14T06:16:06,000 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T06:16:06,003 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T06:16:06,018 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:422830x0, quorum=127.0.0.1:61013, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T06:16:06,019 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:422830x0, quorum=127.0.0.1:61013, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T06:16:06,021 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42283-0x10137fc85b70001 connected 2024-11-14T06:16:06,023 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-14T06:16:06,030 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-14T06:16:06,033 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42283-0x10137fc85b70001, quorum=127.0.0.1:61013, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-14T06:16:06,038 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42283-0x10137fc85b70001, quorum=127.0.0.1:61013, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T06:16:06,038 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42283 2024-11-14T06:16:06,039 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42283 2024-11-14T06:16:06,039 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42283 2024-11-14T06:16:06,040 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42283 2024-11-14T06:16:06,040 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42283 2024-11-14T06:16:06,057 DEBUG [M:0;ef93ef3a83c5:36045 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;ef93ef3a83c5:36045 2024-11-14T06:16:06,058 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/ef93ef3a83c5,36045,1731564965170 2024-11-14T06:16:06,076 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42283-0x10137fc85b70001, quorum=127.0.0.1:61013, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T06:16:06,076 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36045-0x10137fc85b70000, quorum=127.0.0.1:61013, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T06:16:06,078 DEBUG [master/ef93ef3a83c5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36045-0x10137fc85b70000, quorum=127.0.0.1:61013, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/ef93ef3a83c5,36045,1731564965170 2024-11-14T06:16:06,112 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42283-0x10137fc85b70001, quorum=127.0.0.1:61013, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-14T06:16:06,112 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36045-0x10137fc85b70000, quorum=127.0.0.1:61013, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T06:16:06,112 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42283-0x10137fc85b70001, quorum=127.0.0.1:61013, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T06:16:06,113 DEBUG [master/ef93ef3a83c5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36045-0x10137fc85b70000, quorum=127.0.0.1:61013, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-14T06:16:06,114 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/ef93ef3a83c5,36045,1731564965170 from backup master directory 2024-11-14T06:16:06,128 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42283-0x10137fc85b70001, quorum=127.0.0.1:61013, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T06:16:06,128 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36045-0x10137fc85b70000, quorum=127.0.0.1:61013, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/ef93ef3a83c5,36045,1731564965170 2024-11-14T06:16:06,128 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36045-0x10137fc85b70000, quorum=127.0.0.1:61013, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T06:16:06,129 WARN [master/ef93ef3a83c5:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T06:16:06,129 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=ef93ef3a83c5,36045,1731564965170 2024-11-14T06:16:06,132 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-14T06:16:06,134 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-14T06:16:06,185 DEBUG [master/ef93ef3a83c5:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/hbase.id] with ID: f5b31206-2687-4dcd-9a11-4e6c55e0cfc0 2024-11-14T06:16:06,185 DEBUG [master/ef93ef3a83c5:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/.tmp/hbase.id 2024-11-14T06:16:06,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741826_1002 (size=42) 2024-11-14T06:16:06,599 DEBUG [master/ef93ef3a83c5:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/.tmp/hbase.id]:[hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/hbase.id] 2024-11-14T06:16:06,652 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T06:16:06,657 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-14T06:16:06,676 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-11-14T06:16:06,724 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36045-0x10137fc85b70000, quorum=127.0.0.1:61013, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T06:16:06,724 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42283-0x10137fc85b70001, quorum=127.0.0.1:61013, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T06:16:06,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741827_1003 (size=196) 2024-11-14T06:16:06,763 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T06:16:06,767 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-14T06:16:06,789 DEBUG [master/ef93ef3a83c5:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:06,795 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T06:16:06,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741828_1004 (size=1189) 2024-11-14T06:16:06,846 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/MasterData/data/master/store 2024-11-14T06:16:06,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741829_1005 (size=34) 2024-11-14T06:16:07,270 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-14T06:16:07,274 DEBUG [master/ef93ef3a83c5:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T06:16:07,275 DEBUG [master/ef93ef3a83c5:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T06:16:07,275 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T06:16:07,275 DEBUG [master/ef93ef3a83c5:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T06:16:07,277 DEBUG [master/ef93ef3a83c5:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T06:16:07,277 DEBUG [master/ef93ef3a83c5:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T06:16:07,277 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T06:16:07,279 DEBUG [master/ef93ef3a83c5:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731564967275Disabling compacts and flushes for region at 1731564967275Disabling writes for close at 1731564967277 (+2 ms)Writing region close event to WAL at 1731564967277Closed at 1731564967277 2024-11-14T06:16:07,281 WARN [master/ef93ef3a83c5:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/MasterData/data/master/store/.initializing 2024-11-14T06:16:07,281 DEBUG [master/ef93ef3a83c5:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/MasterData/WALs/ef93ef3a83c5,36045,1731564965170 2024-11-14T06:16:07,289 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T06:16:07,304 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ef93ef3a83c5%2C36045%2C1731564965170, suffix=, logDir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/MasterData/WALs/ef93ef3a83c5,36045,1731564965170, archiveDir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/MasterData/oldWALs, maxLogs=10 2024-11-14T06:16:07,329 DEBUG [master/ef93ef3a83c5:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/MasterData/WALs/ef93ef3a83c5,36045,1731564965170/ef93ef3a83c5%2C36045%2C1731564965170.1731564967309, exclude list is [], retry=0 2024-11-14T06:16:07,344 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33461,DS-7f1dcb8e-c4d0-4e38-8a98-6bff453afa6a,DISK] 2024-11-14T06:16:07,347 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-14T06:16:07,378 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/MasterData/WALs/ef93ef3a83c5,36045,1731564965170/ef93ef3a83c5%2C36045%2C1731564965170.1731564967309 2024-11-14T06:16:07,378 DEBUG [master/ef93ef3a83c5:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44151:44151)] 2024-11-14T06:16:07,379 DEBUG [master/ef93ef3a83c5:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-14T06:16:07,379 DEBUG [master/ef93ef3a83c5:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T06:16:07,383 DEBUG [master/ef93ef3a83c5:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T06:16:07,384 DEBUG [master/ef93ef3a83c5:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T06:16:07,415 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T06:16:07,437 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-14T06:16:07,442 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:07,445 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T06:16:07,446 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T06:16:07,451 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-14T06:16:07,451 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:07,453 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T06:16:07,453 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T06:16:07,457 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-14T06:16:07,458 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:07,459 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T06:16:07,460 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T06:16:07,463 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-14T06:16:07,464 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:07,465 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T06:16:07,466 DEBUG [master/ef93ef3a83c5:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T06:16:07,470 DEBUG [master/ef93ef3a83c5:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T06:16:07,471 DEBUG [master/ef93ef3a83c5:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T06:16:07,477 DEBUG [master/ef93ef3a83c5:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T06:16:07,478 DEBUG [master/ef93ef3a83c5:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T06:16:07,482 DEBUG [master/ef93ef3a83c5:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T06:16:07,486 DEBUG [master/ef93ef3a83c5:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T06:16:07,491 DEBUG [master/ef93ef3a83c5:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T06:16:07,493 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71274811, jitterRate=0.06207744777202606}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T06:16:07,500 DEBUG [master/ef93ef3a83c5:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731564967394Initializing all the Stores at 1731564967396 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731564967397 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731564967397Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731564967398 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731564967398Cleaning up temporary data from old regions at 1731564967478 (+80 ms)Region opened successfully at 1731564967500 (+22 ms) 2024-11-14T06:16:07,502 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-14T06:16:07,532 DEBUG [master/ef93ef3a83c5:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20cf113a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ef93ef3a83c5/172.17.0.3:0 2024-11-14T06:16:07,559 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-14T06:16:07,569 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-14T06:16:07,569 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-14T06:16:07,571 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T06:16:07,573 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-14T06:16:07,577 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-14T06:16:07,577 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-14T06:16:07,601 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-14T06:16:07,609 DEBUG [master/ef93ef3a83c5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36045-0x10137fc85b70000, quorum=127.0.0.1:61013, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-14T06:16:07,659 DEBUG [master/ef93ef3a83c5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-14T06:16:07,663 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-14T06:16:07,665 DEBUG [master/ef93ef3a83c5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36045-0x10137fc85b70000, quorum=127.0.0.1:61013, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-14T06:16:07,675 DEBUG [master/ef93ef3a83c5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-14T06:16:07,679 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-14T06:16:07,685 DEBUG [master/ef93ef3a83c5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36045-0x10137fc85b70000, quorum=127.0.0.1:61013, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-14T06:16:07,696 DEBUG [master/ef93ef3a83c5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-14T06:16:07,697 DEBUG [master/ef93ef3a83c5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36045-0x10137fc85b70000, quorum=127.0.0.1:61013, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-14T06:16:07,706 DEBUG [master/ef93ef3a83c5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-14T06:16:07,728 DEBUG [master/ef93ef3a83c5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36045-0x10137fc85b70000, quorum=127.0.0.1:61013, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-14T06:16:07,738 DEBUG [master/ef93ef3a83c5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-14T06:16:07,749 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36045-0x10137fc85b70000, quorum=127.0.0.1:61013, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T06:16:07,749 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42283-0x10137fc85b70001, quorum=127.0.0.1:61013, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T06:16:07,749 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36045-0x10137fc85b70000, quorum=127.0.0.1:61013, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T06:16:07,749 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42283-0x10137fc85b70001, quorum=127.0.0.1:61013, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T06:16:07,751 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=ef93ef3a83c5,36045,1731564965170, sessionid=0x10137fc85b70000, setting cluster-up flag (Was=false) 2024-11-14T06:16:07,780 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42283-0x10137fc85b70001, quorum=127.0.0.1:61013, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T06:16:07,780 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36045-0x10137fc85b70000, quorum=127.0.0.1:61013, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T06:16:07,812 DEBUG [master/ef93ef3a83c5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-14T06:16:07,815 DEBUG [master/ef93ef3a83c5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ef93ef3a83c5,36045,1731564965170 2024-11-14T06:16:07,838 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42283-0x10137fc85b70001, quorum=127.0.0.1:61013, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T06:16:07,838 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36045-0x10137fc85b70000, quorum=127.0.0.1:61013, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T06:16:07,870 DEBUG [master/ef93ef3a83c5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-14T06:16:07,873 DEBUG [master/ef93ef3a83c5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ef93ef3a83c5,36045,1731564965170 2024-11-14T06:16:07,883 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-14T06:16:07,945 INFO [RS:0;ef93ef3a83c5:42283 {}] regionserver.HRegionServer(746): ClusterId : f5b31206-2687-4dcd-9a11-4e6c55e0cfc0 2024-11-14T06:16:07,947 DEBUG [RS:0;ef93ef3a83c5:42283 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-14T06:16:07,959 DEBUG [master/ef93ef3a83c5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T06:16:07,961 DEBUG [RS:0;ef93ef3a83c5:42283 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-14T06:16:07,961 DEBUG [RS:0;ef93ef3a83c5:42283 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-14T06:16:07,969 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-14T06:16:07,971 DEBUG [RS:0;ef93ef3a83c5:42283 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-14T06:16:07,971 DEBUG [RS:0;ef93ef3a83c5:42283 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69103853, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ef93ef3a83c5/172.17.0.3:0 2024-11-14T06:16:07,978 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-14T06:16:07,985 DEBUG [RS:0;ef93ef3a83c5:42283 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;ef93ef3a83c5:42283 2024-11-14T06:16:07,984 DEBUG [master/ef93ef3a83c5:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: ef93ef3a83c5,36045,1731564965170 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-14T06:16:07,988 INFO [RS:0;ef93ef3a83c5:42283 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-14T06:16:07,988 INFO [RS:0;ef93ef3a83c5:42283 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-14T06:16:07,988 DEBUG [RS:0;ef93ef3a83c5:42283 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-14T06:16:07,991 INFO [RS:0;ef93ef3a83c5:42283 {}] regionserver.HRegionServer(2659): reportForDuty to master=ef93ef3a83c5,36045,1731564965170 with port=42283, startcode=1731564965945 2024-11-14T06:16:07,991 DEBUG [master/ef93ef3a83c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/ef93ef3a83c5:0, corePoolSize=5, maxPoolSize=5 2024-11-14T06:16:07,991 DEBUG [master/ef93ef3a83c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/ef93ef3a83c5:0, corePoolSize=5, maxPoolSize=5 2024-11-14T06:16:07,991 DEBUG [master/ef93ef3a83c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/ef93ef3a83c5:0, corePoolSize=5, maxPoolSize=5 2024-11-14T06:16:07,991 DEBUG [master/ef93ef3a83c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/ef93ef3a83c5:0, corePoolSize=5, maxPoolSize=5 2024-11-14T06:16:07,992 DEBUG [master/ef93ef3a83c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/ef93ef3a83c5:0, corePoolSize=10, maxPoolSize=10 2024-11-14T06:16:07,992 DEBUG [master/ef93ef3a83c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/ef93ef3a83c5:0, corePoolSize=1, maxPoolSize=1 2024-11-14T06:16:07,992 DEBUG [master/ef93ef3a83c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/ef93ef3a83c5:0, corePoolSize=2, maxPoolSize=2 2024-11-14T06:16:07,992 DEBUG [master/ef93ef3a83c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/ef93ef3a83c5:0, corePoolSize=1, maxPoolSize=1 2024-11-14T06:16:07,996 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731564997996 2024-11-14T06:16:07,998 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-14T06:16:07,999 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-14T06:16:07,999 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T06:16:07,999 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-14T06:16:08,001 DEBUG [RS:0;ef93ef3a83c5:42283 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-14T06:16:08,002 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-14T06:16:08,003 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-14T06:16:08,003 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-14T06:16:08,003 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-14T06:16:08,005 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:08,003 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T06:16:08,005 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-14T06:16:08,007 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-14T06:16:08,009 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-14T06:16:08,009 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-14T06:16:08,011 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-14T06:16:08,011 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-14T06:16:08,013 DEBUG [master/ef93ef3a83c5:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/ef93ef3a83c5:0:becomeActiveMaster-HFileCleaner.large.0-1731564968013,5,FailOnTimeoutGroup] 2024-11-14T06:16:08,016 DEBUG [master/ef93ef3a83c5:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/ef93ef3a83c5:0:becomeActiveMaster-HFileCleaner.small.0-1731564968013,5,FailOnTimeoutGroup] 2024-11-14T06:16:08,016 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T06:16:08,016 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-14T06:16:08,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741831_1007 (size=1321) 2024-11-14T06:16:08,018 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-14T06:16:08,018 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-14T06:16:08,020 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-14T06:16:08,021 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02 2024-11-14T06:16:08,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741832_1008 (size=32) 2024-11-14T06:16:08,039 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41639, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-14T06:16:08,045 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36045 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:08,047 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36045 {}] master.ServerManager(517): Registering regionserver=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:08,060 DEBUG [RS:0;ef93ef3a83c5:42283 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02 2024-11-14T06:16:08,060 DEBUG [RS:0;ef93ef3a83c5:42283 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35361 2024-11-14T06:16:08,060 DEBUG [RS:0;ef93ef3a83c5:42283 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-14T06:16:08,075 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36045-0x10137fc85b70000, quorum=127.0.0.1:61013, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T06:16:08,076 DEBUG [RS:0;ef93ef3a83c5:42283 {}] zookeeper.ZKUtil(111): regionserver:42283-0x10137fc85b70001, quorum=127.0.0.1:61013, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:08,076 WARN [RS:0;ef93ef3a83c5:42283 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T06:16:08,076 INFO [RS:0;ef93ef3a83c5:42283 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T06:16:08,077 DEBUG [RS:0;ef93ef3a83c5:42283 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/WALs/ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:08,078 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ef93ef3a83c5,42283,1731564965945] 2024-11-14T06:16:08,105 INFO [RS:0;ef93ef3a83c5:42283 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-14T06:16:08,120 INFO [RS:0;ef93ef3a83c5:42283 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-14T06:16:08,125 INFO [RS:0;ef93ef3a83c5:42283 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T06:16:08,125 INFO [RS:0;ef93ef3a83c5:42283 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T06:16:08,126 INFO [RS:0;ef93ef3a83c5:42283 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-14T06:16:08,132 INFO [RS:0;ef93ef3a83c5:42283 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-14T06:16:08,133 INFO [RS:0;ef93ef3a83c5:42283 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-14T06:16:08,134 DEBUG [RS:0;ef93ef3a83c5:42283 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ef93ef3a83c5:0, corePoolSize=1, maxPoolSize=1 2024-11-14T06:16:08,134 DEBUG [RS:0;ef93ef3a83c5:42283 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ef93ef3a83c5:0, corePoolSize=1, maxPoolSize=1 2024-11-14T06:16:08,134 DEBUG [RS:0;ef93ef3a83c5:42283 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ef93ef3a83c5:0, corePoolSize=1, maxPoolSize=1 2024-11-14T06:16:08,134 DEBUG [RS:0;ef93ef3a83c5:42283 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0, corePoolSize=1, maxPoolSize=1 2024-11-14T06:16:08,135 DEBUG [RS:0;ef93ef3a83c5:42283 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ef93ef3a83c5:0, corePoolSize=1, maxPoolSize=1 2024-11-14T06:16:08,135 DEBUG [RS:0;ef93ef3a83c5:42283 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ef93ef3a83c5:0, corePoolSize=2, maxPoolSize=2 2024-11-14T06:16:08,135 DEBUG [RS:0;ef93ef3a83c5:42283 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ef93ef3a83c5:0, corePoolSize=1, maxPoolSize=1 2024-11-14T06:16:08,135 DEBUG [RS:0;ef93ef3a83c5:42283 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ef93ef3a83c5:0, corePoolSize=1, maxPoolSize=1 2024-11-14T06:16:08,135 DEBUG [RS:0;ef93ef3a83c5:42283 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ef93ef3a83c5:0, corePoolSize=1, maxPoolSize=1 2024-11-14T06:16:08,136 DEBUG [RS:0;ef93ef3a83c5:42283 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ef93ef3a83c5:0, corePoolSize=1, maxPoolSize=1 2024-11-14T06:16:08,136 DEBUG [RS:0;ef93ef3a83c5:42283 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ef93ef3a83c5:0, corePoolSize=1, maxPoolSize=1 2024-11-14T06:16:08,136 DEBUG [RS:0;ef93ef3a83c5:42283 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ef93ef3a83c5:0, corePoolSize=1, maxPoolSize=1 2024-11-14T06:16:08,136 DEBUG [RS:0;ef93ef3a83c5:42283 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ef93ef3a83c5:0, corePoolSize=3, maxPoolSize=3 2024-11-14T06:16:08,136 DEBUG [RS:0;ef93ef3a83c5:42283 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0, corePoolSize=3, maxPoolSize=3 2024-11-14T06:16:08,137 INFO [RS:0;ef93ef3a83c5:42283 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T06:16:08,138 INFO [RS:0;ef93ef3a83c5:42283 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T06:16:08,138 INFO [RS:0;ef93ef3a83c5:42283 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T06:16:08,138 INFO [RS:0;ef93ef3a83c5:42283 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-14T06:16:08,138 INFO [RS:0;ef93ef3a83c5:42283 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-14T06:16:08,138 INFO [RS:0;ef93ef3a83c5:42283 {}] hbase.ChoreService(168): Chore ScheduledChore name=ef93ef3a83c5,42283,1731564965945-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T06:16:08,159 INFO [RS:0;ef93ef3a83c5:42283 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-14T06:16:08,160 INFO [RS:0;ef93ef3a83c5:42283 {}] hbase.ChoreService(168): Chore ScheduledChore name=ef93ef3a83c5,42283,1731564965945-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T06:16:08,160 INFO [RS:0;ef93ef3a83c5:42283 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T06:16:08,160 INFO [RS:0;ef93ef3a83c5:42283 {}] regionserver.Replication(171): ef93ef3a83c5,42283,1731564965945 started 2024-11-14T06:16:08,185 INFO [RS:0;ef93ef3a83c5:42283 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T06:16:08,185 INFO [RS:0;ef93ef3a83c5:42283 {}] regionserver.HRegionServer(1482): Serving as ef93ef3a83c5,42283,1731564965945, RpcServer on ef93ef3a83c5/172.17.0.3:42283, sessionid=0x10137fc85b70001 2024-11-14T06:16:08,186 DEBUG [RS:0;ef93ef3a83c5:42283 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-14T06:16:08,186 DEBUG [RS:0;ef93ef3a83c5:42283 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:08,186 DEBUG [RS:0;ef93ef3a83c5:42283 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ef93ef3a83c5,42283,1731564965945' 2024-11-14T06:16:08,187 DEBUG [RS:0;ef93ef3a83c5:42283 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-14T06:16:08,188 DEBUG [RS:0;ef93ef3a83c5:42283 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-14T06:16:08,189 DEBUG [RS:0;ef93ef3a83c5:42283 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-14T06:16:08,189 DEBUG [RS:0;ef93ef3a83c5:42283 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-14T06:16:08,189 DEBUG [RS:0;ef93ef3a83c5:42283 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:08,189 DEBUG [RS:0;ef93ef3a83c5:42283 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ef93ef3a83c5,42283,1731564965945' 2024-11-14T06:16:08,189 DEBUG [RS:0;ef93ef3a83c5:42283 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-14T06:16:08,190 DEBUG [RS:0;ef93ef3a83c5:42283 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-14T06:16:08,191 DEBUG [RS:0;ef93ef3a83c5:42283 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-14T06:16:08,191 INFO [RS:0;ef93ef3a83c5:42283 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-14T06:16:08,192 INFO [RS:0;ef93ef3a83c5:42283 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-14T06:16:08,302 INFO [RS:0;ef93ef3a83c5:42283 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T06:16:08,305 INFO [RS:0;ef93ef3a83c5:42283 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ef93ef3a83c5%2C42283%2C1731564965945, suffix=, logDir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/WALs/ef93ef3a83c5,42283,1731564965945, archiveDir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/oldWALs, maxLogs=32 2024-11-14T06:16:08,322 DEBUG [RS:0;ef93ef3a83c5:42283 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/WALs/ef93ef3a83c5,42283,1731564965945/ef93ef3a83c5%2C42283%2C1731564965945.1731564968308, exclude list is [], retry=0 2024-11-14T06:16:08,328 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33461,DS-7f1dcb8e-c4d0-4e38-8a98-6bff453afa6a,DISK] 2024-11-14T06:16:08,334 INFO [RS:0;ef93ef3a83c5:42283 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/WALs/ef93ef3a83c5,42283,1731564965945/ef93ef3a83c5%2C42283%2C1731564965945.1731564968308 2024-11-14T06:16:08,334 DEBUG [RS:0;ef93ef3a83c5:42283 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44151:44151)] 2024-11-14T06:16:08,436 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T06:16:08,439 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-14T06:16:08,441 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T06:16:08,441 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:08,443 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T06:16:08,443 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-14T06:16:08,446 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T06:16:08,446 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:08,448 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T06:16:08,448 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-14T06:16:08,451 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T06:16:08,451 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:08,452 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T06:16:08,452 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-14T06:16:08,455 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T06:16:08,455 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:08,456 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T06:16:08,456 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T06:16:08,458 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/hbase/meta/1588230740 2024-11-14T06:16:08,458 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/hbase/meta/1588230740 2024-11-14T06:16:08,461 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T06:16:08,461 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T06:16:08,462 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T06:16:08,464 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T06:16:08,467 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T06:16:08,468 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71628555, jitterRate=0.06734864413738251}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T06:16:08,471 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731564968436Initializing all the Stores at 1731564968438 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731564968438Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731564968438Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731564968438Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731564968438Cleaning up temporary data from old regions at 1731564968461 (+23 ms)Region opened successfully at 1731564968471 (+10 ms) 2024-11-14T06:16:08,471 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T06:16:08,471 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T06:16:08,471 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T06:16:08,471 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T06:16:08,471 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T06:16:08,473 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T06:16:08,473 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731564968471Disabling compacts and flushes for region at 1731564968471Disabling writes for close at 1731564968471Writing region close event to WAL at 1731564968472 (+1 ms)Closed at 1731564968473 (+1 ms) 2024-11-14T06:16:08,476 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T06:16:08,476 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-14T06:16:08,482 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-14T06:16:08,490 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T06:16:08,492 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-14T06:16:08,649 DEBUG [ef93ef3a83c5:36045 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-14T06:16:08,662 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:08,667 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ef93ef3a83c5,42283,1731564965945, state=OPENING 2024-11-14T06:16:08,680 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-14T06:16:08,691 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42283-0x10137fc85b70001, quorum=127.0.0.1:61013, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T06:16:08,691 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36045-0x10137fc85b70000, quorum=127.0.0.1:61013, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T06:16:08,692 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T06:16:08,692 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T06:16:08,694 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T06:16:08,697 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=ef93ef3a83c5,42283,1731564965945}] 2024-11-14T06:16:08,876 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-14T06:16:08,880 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41621, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-14T06:16:08,890 INFO [RS_OPEN_META-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-14T06:16:08,890 INFO [RS_OPEN_META-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T06:16:08,891 INFO [RS_OPEN_META-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-14T06:16:08,894 INFO [RS_OPEN_META-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ef93ef3a83c5%2C42283%2C1731564965945.meta, suffix=.meta, logDir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/WALs/ef93ef3a83c5,42283,1731564965945, archiveDir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/oldWALs, maxLogs=32 2024-11-14T06:16:08,911 DEBUG [RS_OPEN_META-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/WALs/ef93ef3a83c5,42283,1731564965945/ef93ef3a83c5%2C42283%2C1731564965945.meta.1731564968897.meta, exclude list is [], retry=0 2024-11-14T06:16:08,915 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33461,DS-7f1dcb8e-c4d0-4e38-8a98-6bff453afa6a,DISK] 2024-11-14T06:16:08,919 INFO [RS_OPEN_META-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/WALs/ef93ef3a83c5,42283,1731564965945/ef93ef3a83c5%2C42283%2C1731564965945.meta.1731564968897.meta 2024-11-14T06:16:08,919 DEBUG [RS_OPEN_META-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44151:44151)] 2024-11-14T06:16:08,920 DEBUG [RS_OPEN_META-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-14T06:16:08,922 DEBUG [RS_OPEN_META-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-14T06:16:08,925 DEBUG [RS_OPEN_META-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-14T06:16:08,929 INFO [RS_OPEN_META-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-14T06:16:08,933 DEBUG [RS_OPEN_META-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-14T06:16:08,934 DEBUG [RS_OPEN_META-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T06:16:08,934 DEBUG [RS_OPEN_META-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-14T06:16:08,934 DEBUG [RS_OPEN_META-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-14T06:16:08,938 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-14T06:16:08,939 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T06:16:08,939 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:08,940 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T06:16:08,941 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-14T06:16:08,942 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T06:16:08,942 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:08,943 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T06:16:08,944 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-14T06:16:08,945 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T06:16:08,945 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:08,946 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T06:16:08,946 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-14T06:16:08,948 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T06:16:08,948 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:08,949 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T06:16:08,949 DEBUG [RS_OPEN_META-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T06:16:08,951 DEBUG [RS_OPEN_META-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/hbase/meta/1588230740 2024-11-14T06:16:08,954 DEBUG [RS_OPEN_META-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/hbase/meta/1588230740 2024-11-14T06:16:08,957 DEBUG [RS_OPEN_META-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T06:16:08,958 DEBUG [RS_OPEN_META-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T06:16:08,959 DEBUG [RS_OPEN_META-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T06:16:08,962 DEBUG [RS_OPEN_META-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T06:16:08,964 INFO [RS_OPEN_META-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63460186, jitterRate=-0.05436953902244568}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T06:16:08,964 DEBUG [RS_OPEN_META-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-14T06:16:08,966 DEBUG [RS_OPEN_META-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731564968935Writing region info on filesystem at 1731564968935Initializing all the Stores at 1731564968937 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731564968937Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731564968937Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731564968937Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731564968937Cleaning up temporary data from old regions at 1731564968958 (+21 ms)Running coprocessor post-open hooks at 1731564968964 (+6 ms)Region opened successfully at 1731564968966 (+2 ms) 2024-11-14T06:16:08,974 INFO [RS_OPEN_META-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731564968865 2024-11-14T06:16:08,985 DEBUG [RS_OPEN_META-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-14T06:16:08,986 INFO [RS_OPEN_META-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-14T06:16:08,988 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:08,990 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ef93ef3a83c5,42283,1731564965945, state=OPEN 2024-11-14T06:16:09,121 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36045-0x10137fc85b70000, quorum=127.0.0.1:61013, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T06:16:09,121 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42283-0x10137fc85b70001, quorum=127.0.0.1:61013, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T06:16:09,121 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T06:16:09,122 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T06:16:09,122 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:09,131 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-14T06:16:09,132 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ef93ef3a83c5,42283,1731564965945 in 426 msec 2024-11-14T06:16:09,138 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-14T06:16:09,139 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 651 msec 2024-11-14T06:16:09,140 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T06:16:09,140 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-14T06:16:09,157 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:16:09,159 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:16:09,176 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:16:09,178 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43923, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:16:09,204 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2870 sec 2024-11-14T06:16:09,204 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731564969204, completionTime=-1 2024-11-14T06:16:09,207 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-14T06:16:09,207 DEBUG [master/ef93ef3a83c5:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-14T06:16:09,236 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-14T06:16:09,236 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731565029236 2024-11-14T06:16:09,236 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731565089236 2024-11-14T06:16:09,236 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 28 msec 2024-11-14T06:16:09,239 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ef93ef3a83c5,36045,1731564965170-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T06:16:09,240 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ef93ef3a83c5,36045,1731564965170-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T06:16:09,240 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ef93ef3a83c5,36045,1731564965170-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T06:16:09,242 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-ef93ef3a83c5:36045, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T06:16:09,243 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-14T06:16:09,243 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-14T06:16:09,252 DEBUG [master/ef93ef3a83c5:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-14T06:16:09,274 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.143sec 2024-11-14T06:16:09,275 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-14T06:16:09,276 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-14T06:16:09,277 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-14T06:16:09,278 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-14T06:16:09,278 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-14T06:16:09,279 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ef93ef3a83c5,36045,1731564965170-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T06:16:09,280 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ef93ef3a83c5,36045,1731564965170-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-14T06:16:09,291 DEBUG [master/ef93ef3a83c5:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-14T06:16:09,292 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-14T06:16:09,292 INFO [master/ef93ef3a83c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ef93ef3a83c5,36045,1731564965170-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T06:16:09,354 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4de0fcb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:16:09,356 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-14T06:16:09,356 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-14T06:16:09,359 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:16:09,362 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:16:09,370 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:16:09,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:16:09,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:16:09,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@993125d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:16:09,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:16:09,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:16:09,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:16:09,380 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33618, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:16:09,383 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16f25d2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:16:09,383 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:16:09,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:16:09,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:16:09,395 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46650, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:16:09,416 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=ef93ef3a83c5,36045,1731564965170 2024-11-14T06:16:09,435 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=231, ProcessCount=11, AvailableMemoryMB=3518 2024-11-14T06:16:09,448 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T06:16:09,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is ef93ef3a83c5,36045,1731564965170 2024-11-14T06:16:09,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@35874ff9 2024-11-14T06:16:09,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T06:16:09,485 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33626, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T06:16:09,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T06:16:09,492 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T06:16:09,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T06:16:09,509 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T06:16:09,511 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-14T06:16:09,512 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:09,514 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T06:16:09,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T06:16:09,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741835_1011 (size=1061) 2024-11-14T06:16:09,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T06:16:09,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T06:16:09,934 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02 2024-11-14T06:16:09,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741836_1012 (size=53) 2024-11-14T06:16:10,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T06:16:10,350 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T06:16:10,350 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing d7851abd86d644f97c54179c322b1c1e, disabling compactions & flushes 2024-11-14T06:16:10,350 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:10,350 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:10,351 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. after waiting 0 ms 2024-11-14T06:16:10,351 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:10,351 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:10,351 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for d7851abd86d644f97c54179c322b1c1e: Waiting for close lock at 1731564970350Disabling compacts and flushes for region at 1731564970350Disabling writes for close at 1731564970351 (+1 ms)Writing region close event to WAL at 1731564970351Closed at 1731564970351 2024-11-14T06:16:10,355 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T06:16:10,362 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731564970355"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731564970355"}]},"ts":"1731564970355"} 2024-11-14T06:16:10,366 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T06:16:10,368 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T06:16:10,371 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731564970368"}]},"ts":"1731564970368"} 2024-11-14T06:16:10,376 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T06:16:10,378 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d7851abd86d644f97c54179c322b1c1e, ASSIGN}] 2024-11-14T06:16:10,381 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d7851abd86d644f97c54179c322b1c1e, ASSIGN 2024-11-14T06:16:10,383 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d7851abd86d644f97c54179c322b1c1e, ASSIGN; state=OFFLINE, location=ef93ef3a83c5,42283,1731564965945; forceNewPlan=false, retain=false 2024-11-14T06:16:10,535 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=d7851abd86d644f97c54179c322b1c1e, regionState=OPENING, regionLocation=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:10,544 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d7851abd86d644f97c54179c322b1c1e, ASSIGN because future has completed 2024-11-14T06:16:10,546 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945}] 2024-11-14T06:16:10,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T06:16:10,710 INFO [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:10,711 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} 2024-11-14T06:16:10,711 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d7851abd86d644f97c54179c322b1c1e 2024-11-14T06:16:10,711 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T06:16:10,711 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for d7851abd86d644f97c54179c322b1c1e 2024-11-14T06:16:10,712 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for d7851abd86d644f97c54179c322b1c1e 2024-11-14T06:16:10,714 INFO [StoreOpener-d7851abd86d644f97c54179c322b1c1e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d7851abd86d644f97c54179c322b1c1e 2024-11-14T06:16:10,718 INFO [StoreOpener-d7851abd86d644f97c54179c322b1c1e-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T06:16:10,718 INFO [StoreOpener-d7851abd86d644f97c54179c322b1c1e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d7851abd86d644f97c54179c322b1c1e columnFamilyName A 2024-11-14T06:16:10,718 DEBUG [StoreOpener-d7851abd86d644f97c54179c322b1c1e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:10,719 INFO [StoreOpener-d7851abd86d644f97c54179c322b1c1e-1 {}] regionserver.HStore(327): Store=d7851abd86d644f97c54179c322b1c1e/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T06:16:10,720 INFO [StoreOpener-d7851abd86d644f97c54179c322b1c1e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d7851abd86d644f97c54179c322b1c1e 2024-11-14T06:16:10,722 INFO [StoreOpener-d7851abd86d644f97c54179c322b1c1e-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T06:16:10,722 INFO [StoreOpener-d7851abd86d644f97c54179c322b1c1e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d7851abd86d644f97c54179c322b1c1e columnFamilyName B 2024-11-14T06:16:10,722 DEBUG [StoreOpener-d7851abd86d644f97c54179c322b1c1e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:10,723 INFO [StoreOpener-d7851abd86d644f97c54179c322b1c1e-1 {}] regionserver.HStore(327): Store=d7851abd86d644f97c54179c322b1c1e/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T06:16:10,723 INFO [StoreOpener-d7851abd86d644f97c54179c322b1c1e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d7851abd86d644f97c54179c322b1c1e 2024-11-14T06:16:10,725 INFO [StoreOpener-d7851abd86d644f97c54179c322b1c1e-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T06:16:10,725 INFO [StoreOpener-d7851abd86d644f97c54179c322b1c1e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d7851abd86d644f97c54179c322b1c1e columnFamilyName C 2024-11-14T06:16:10,725 DEBUG [StoreOpener-d7851abd86d644f97c54179c322b1c1e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:10,726 INFO [StoreOpener-d7851abd86d644f97c54179c322b1c1e-1 {}] regionserver.HStore(327): Store=d7851abd86d644f97c54179c322b1c1e/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T06:16:10,727 INFO [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:10,727 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for d7851abd86d644f97c54179c322b1c1e 2024-11-14T06:16:10,728 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e 2024-11-14T06:16:10,729 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e 2024-11-14T06:16:10,731 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for d7851abd86d644f97c54179c322b1c1e 2024-11-14T06:16:10,731 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for d7851abd86d644f97c54179c322b1c1e 2024-11-14T06:16:10,732 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T06:16:10,734 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for d7851abd86d644f97c54179c322b1c1e 2024-11-14T06:16:10,738 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T06:16:10,738 INFO [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened d7851abd86d644f97c54179c322b1c1e; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68292269, jitterRate=0.017634108662605286}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T06:16:10,739 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d7851abd86d644f97c54179c322b1c1e 2024-11-14T06:16:10,739 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for d7851abd86d644f97c54179c322b1c1e: Running coprocessor pre-open hook at 1731564970712Writing region info on filesystem at 1731564970712Initializing all the Stores at 1731564970714 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731564970714Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731564970714Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731564970714Cleaning up temporary data from old regions at 1731564970731 (+17 ms)Running coprocessor post-open hooks at 1731564970739 (+8 ms)Region opened successfully at 1731564970739 2024-11-14T06:16:10,741 INFO [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., pid=6, masterSystemTime=1731564970703 2024-11-14T06:16:10,745 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:10,745 INFO [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:10,747 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=d7851abd86d644f97c54179c322b1c1e, regionState=OPEN, openSeqNum=2, regionLocation=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:10,751 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 because future has completed 2024-11-14T06:16:10,759 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-14T06:16:10,759 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 in 208 msec 2024-11-14T06:16:10,765 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-14T06:16:10,765 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d7851abd86d644f97c54179c322b1c1e, ASSIGN in 381 msec 2024-11-14T06:16:10,767 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T06:16:10,767 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731564970767"}]},"ts":"1731564970767"} 2024-11-14T06:16:10,771 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T06:16:10,773 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T06:16:10,777 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2770 sec 2024-11-14T06:16:11,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T06:16:11,680 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:16:11,687 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3de8c6e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:16:11,687 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:16:11,687 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:16:11,690 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:16:11,691 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:16:11,691 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:16:11,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20fa5eed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:16:11,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:16:11,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:16:11,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:16:11,696 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33656, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:16:11,699 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a66f173, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:16:11,699 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:16:11,702 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:16:11,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:16:11,716 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46658, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:16:11,720 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T06:16:11,723 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is ef93ef3a83c5,36045,1731564965170 2024-11-14T06:16:11,723 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1c52dcb8 2024-11-14T06:16:11,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T06:16:11,730 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33662, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T06:16:11,745 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@557765b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:16:11,745 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:16:11,745 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:16:11,748 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:16:11,749 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:16:11,750 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:16:11,750 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43b9dd8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:16:11,750 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:16:11,750 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:16:11,751 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:16:11,754 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33666, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:16:11,756 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ff1f1f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:16:11,760 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@707437bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:16:11,761 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:16:11,761 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:16:11,766 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:16:11,768 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:16:11,768 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:16:11,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ec01134, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:16:11,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:16:11,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:16:11,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:16:11,799 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33678, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:16:11,801 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b37f010, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:16:11,813 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f6c2394, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:16:11,813 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:16:11,814 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:16:11,820 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:16:11,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:16:11,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:16:11,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@90ab23d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:16:11,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:16:11,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:16:11,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:16:11,826 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33700, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:16:11,828 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2287fe54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:16:11,829 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@120c1a44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:16:11,829 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:16:11,829 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:16:11,835 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:16:11,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:16:11,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:16:11,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b0b2729, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:16:11,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:16:11,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:16:11,845 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:16:11,847 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33714, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:16:11,849 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c42c597, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:16:11,850 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e2eb13d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:16:11,850 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:16:11,851 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:16:11,852 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:16:11,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:16:11,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:16:11,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ac918cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:16:11,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:16:11,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:16:11,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:16:11,856 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33724, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:16:11,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d230d79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:16:11,860 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73fa3b57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:16:11,861 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:16:11,861 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:16:11,863 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:16:11,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:16:11,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:16:11,864 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29adf99b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:16:11,864 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:16:11,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:16:11,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:16:11,867 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33744, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:16:11,868 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f58ce45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:16:11,869 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@396119d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:16:11,870 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:16:11,870 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:16:11,871 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:16:11,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:16:11,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:16:11,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f83cf40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:16:11,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:16:11,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:16:11,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:16:11,875 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33470, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:16:11,876 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@605ecfde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:16:11,877 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23ee0896, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:16:11,877 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:16:11,878 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:16:11,879 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:16:11,879 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:16:11,880 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:16:11,880 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d0e529e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:16:11,880 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:16:11,880 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:16:11,881 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:16:11,882 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33482, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:16:11,884 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7aaa86d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:16:11,885 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d2dce0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:16:11,885 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:16:11,885 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:16:11,886 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:16:11,888 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:16:11,888 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:16:11,889 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d25ecbc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:16:11,889 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:16:11,889 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:16:11,890 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:16:11,891 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33498, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:16:11,892 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a74f818, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:16:11,898 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:16:11,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:16:11,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:16:11,902 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58292, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:16:11,909 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:16:11,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2] 2024-11-14T06:16:11,912 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:16:11,917 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:16:11,917 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:16:11,921 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:16:11,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:16:11,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:16:11,925 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58294, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:16:11,925 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:16:11,926 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:16:11,929 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:16:11,929 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58300, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:16:11,930 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:16:11,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:16:11,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2] 2024-11-14T06:16:11,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:16:11,935 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T06:16:11,936 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:16:11,936 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:16:11,940 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58322, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:16:11,941 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58308, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:16:11,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:16:11,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:16:11,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2] 2024-11-14T06:16:11,944 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:16:11,945 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:16:11,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2] 2024-11-14T06:16:11,947 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58338, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:16:11,948 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2] 2024-11-14T06:16:11,949 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:16:11,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:16:11,951 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:16:11,951 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:16:11,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:16:11,951 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58350, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:16:11,954 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58364, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:16:11,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-14T06:16:11,955 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58366, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:16:11,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2] 2024-11-14T06:16:11,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2] 2024-11-14T06:16:11,959 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T06:16:11,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2] 2024-11-14T06:16:11,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T06:16:11,965 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T06:16:11,967 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T06:16:11,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2] 2024-11-14T06:16:12,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on d7851abd86d644f97c54179c322b1c1e 2024-11-14T06:16:12,056 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d7851abd86d644f97c54179c322b1c1e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T06:16:12,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T06:16:12,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=A 2024-11-14T06:16:12,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:12,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=B 2024-11-14T06:16:12,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:12,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=C 2024-11-14T06:16:12,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:12,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T06:16:12,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:12,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. as already flushing 2024-11-14T06:16:12,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:12,186 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:12,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:12,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:12,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:12,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:12,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565032213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:12,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565032217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:12,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:12,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565032231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:12,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:12,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565032236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:12,268 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/af35530d29884ab99f3e8671e990cf3e is 50, key is test_row_0/A:col10/1731564972012/Put/seqid=0 2024-11-14T06:16:12,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T06:16:12,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:12,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565032246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:12,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:12,311 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:12,311 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:12,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:12,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:12,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:12,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:12,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:12,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:12,313 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:12,313 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:12,313 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:12,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:12,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:12,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:12,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741837_1013 (size=12001) 2024-11-14T06:16:12,373 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/af35530d29884ab99f3e8671e990cf3e 2024-11-14T06:16:12,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T06:16:12,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:12,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. as already flushing 2024-11-14T06:16:12,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:12,398 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:12,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:12,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:12,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:12,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565032420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:12,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:12,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565032421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:12,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:12,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565032424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:12,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:12,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:12,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565032425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:12,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:12,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:12,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:12,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:12,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:12,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:12,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565032426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:12,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:12,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:12,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:12,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:12,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:12,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:12,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:12,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:12,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:12,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T06:16:12,564 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/1f563e6300eb4128b97d7b2e5212cc22 is 50, key is test_row_0/B:col10/1731564972012/Put/seqid=0 2024-11-14T06:16:12,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:12,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. as already flushing 2024-11-14T06:16:12,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:12,577 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:12,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:12,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:12,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T06:16:12,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741838_1014 (size=12001) 2024-11-14T06:16:12,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:12,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565032645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:12,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:12,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:12,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:12,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:12,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565032643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:12,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:12,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:12,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:12,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:12,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565032645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:12,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:12,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:12,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:12,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:12,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565032644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:12,654 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:12,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:12,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:12,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:12,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565032647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:12,661 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:12,661 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:12,661 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:12,748 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T06:16:12,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:12,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. as already flushing 2024-11-14T06:16:12,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:12,749 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:12,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:12,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:12,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T06:16:12,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:12,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. as already flushing 2024-11-14T06:16:12,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:12,906 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:12,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:12,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:12,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:12,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565032958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:12,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:12,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:12,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:12,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:12,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565032958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:12,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:12,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:12,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:12,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:12,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565032959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:12,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:12,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:12,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:12,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:12,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565032968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:12,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:12,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565032969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:12,983 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:12,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:12,983 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:12,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:12,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:12,984 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:13,053 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/1f563e6300eb4128b97d7b2e5212cc22 2024-11-14T06:16:13,063 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T06:16:13,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:13,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. as already flushing 2024-11-14T06:16:13,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:13,063 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:13,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:13,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:13,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T06:16:13,134 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/0a933fe3e7754ed994c3e0ff2d4e1b33 is 50, key is test_row_0/C:col10/1731564972012/Put/seqid=0 2024-11-14T06:16:13,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741839_1015 (size=12001) 2024-11-14T06:16:13,220 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T06:16:13,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:13,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. as already flushing 2024-11-14T06:16:13,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:13,221 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:13,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:13,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:13,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T06:16:13,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:13,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. as already flushing 2024-11-14T06:16:13,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:13,377 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:13,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:13,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:13,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:13,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565033479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:13,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:13,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565033480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:13,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:13,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:13,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:13,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:13,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565033480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:13,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:13,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:13,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:13,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:13,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565033488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:13,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:13,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565033498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:13,516 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:13,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:13,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:13,516 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:13,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:13,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:13,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:13,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:13,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:13,537 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T06:16:13,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:13,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. as already flushing 2024-11-14T06:16:13,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:13,539 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:13,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:13,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:13,557 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/0a933fe3e7754ed994c3e0ff2d4e1b33 2024-11-14T06:16:13,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/af35530d29884ab99f3e8671e990cf3e as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/af35530d29884ab99f3e8671e990cf3e 2024-11-14T06:16:13,607 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/af35530d29884ab99f3e8671e990cf3e, entries=150, sequenceid=14, filesize=11.7 K 2024-11-14T06:16:13,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/1f563e6300eb4128b97d7b2e5212cc22 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/1f563e6300eb4128b97d7b2e5212cc22 2024-11-14T06:16:13,648 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/1f563e6300eb4128b97d7b2e5212cc22, entries=150, sequenceid=14, filesize=11.7 K 2024-11-14T06:16:13,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/0a933fe3e7754ed994c3e0ff2d4e1b33 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/0a933fe3e7754ed994c3e0ff2d4e1b33 2024-11-14T06:16:13,677 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/0a933fe3e7754ed994c3e0ff2d4e1b33, entries=150, sequenceid=14, filesize=11.7 K 2024-11-14T06:16:13,687 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for d7851abd86d644f97c54179c322b1c1e in 1630ms, sequenceid=14, compaction requested=false 2024-11-14T06:16:13,688 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-14T06:16:13,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d7851abd86d644f97c54179c322b1c1e: 2024-11-14T06:16:13,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T06:16:13,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:13,703 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing d7851abd86d644f97c54179c322b1c1e 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-14T06:16:13,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=A 2024-11-14T06:16:13,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:13,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=B 2024-11-14T06:16:13,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:13,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=C 2024-11-14T06:16:13,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:13,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/3f3f93eafcd94ee9b649a4b323167c3c is 50, key is test_row_0/A:col10/1731564972228/Put/seqid=0 2024-11-14T06:16:13,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741840_1016 (size=12001) 2024-11-14T06:16:14,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T06:16:14,197 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/3f3f93eafcd94ee9b649a4b323167c3c 2024-11-14T06:16:14,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/20c53261f4ef42a2b16bc9f1f6198b4b is 50, key is test_row_0/B:col10/1731564972228/Put/seqid=0 2024-11-14T06:16:14,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741841_1017 (size=12001) 2024-11-14T06:16:14,276 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/20c53261f4ef42a2b16bc9f1f6198b4b 2024-11-14T06:16:14,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/16213f65774845a0b74d3d15ea6ae263 is 50, key is test_row_0/C:col10/1731564972228/Put/seqid=0 2024-11-14T06:16:14,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741842_1018 (size=12001) 2024-11-14T06:16:14,449 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T06:16:14,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. as already flushing 2024-11-14T06:16:14,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on d7851abd86d644f97c54179c322b1c1e 2024-11-14T06:16:14,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:14,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565034588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:14,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:14,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:14,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:14,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:14,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565034596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:14,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:14,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:14,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:14,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:14,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565034607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:14,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:14,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:14,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:14,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:14,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565034613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:14,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:14,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:14,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:14,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:14,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565034613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:14,642 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:14,642 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:14,642 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:14,681 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-14T06:16:14,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:14,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565034729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:14,734 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:14,734 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:14,734 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:14,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:14,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565034729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:14,735 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:14,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:14,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:14,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:14,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565034729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:14,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:14,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:14,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:14,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:14,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565034730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:14,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:14,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:14,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:14,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:14,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565034748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:14,762 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:14,762 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:14,762 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:14,764 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/16213f65774845a0b74d3d15ea6ae263 2024-11-14T06:16:14,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/3f3f93eafcd94ee9b649a4b323167c3c as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/3f3f93eafcd94ee9b649a4b323167c3c 2024-11-14T06:16:14,816 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/3f3f93eafcd94ee9b649a4b323167c3c, entries=150, sequenceid=33, filesize=11.7 K 2024-11-14T06:16:14,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/20c53261f4ef42a2b16bc9f1f6198b4b as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/20c53261f4ef42a2b16bc9f1f6198b4b 2024-11-14T06:16:14,874 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/20c53261f4ef42a2b16bc9f1f6198b4b, entries=150, sequenceid=33, filesize=11.7 K 2024-11-14T06:16:14,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/16213f65774845a0b74d3d15ea6ae263 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/16213f65774845a0b74d3d15ea6ae263 2024-11-14T06:16:14,898 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/16213f65774845a0b74d3d15ea6ae263, entries=150, sequenceid=33, filesize=11.7 K 2024-11-14T06:16:14,902 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for d7851abd86d644f97c54179c322b1c1e in 1199ms, sequenceid=33, compaction requested=false 2024-11-14T06:16:14,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for d7851abd86d644f97c54179c322b1c1e: 2024-11-14T06:16:14,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:14,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-14T06:16:14,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-14T06:16:14,912 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-14T06:16:14,918 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9390 sec 2024-11-14T06:16:14,918 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.9770 sec 2024-11-14T06:16:14,958 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d7851abd86d644f97c54179c322b1c1e 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-14T06:16:14,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=A 2024-11-14T06:16:14,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:14,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=B 2024-11-14T06:16:14,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:14,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=C 2024-11-14T06:16:14,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:14,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on d7851abd86d644f97c54179c322b1c1e 2024-11-14T06:16:14,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/95788cae5bb34c848cfa2742fc1a674f is 50, key is test_row_0/A:col10/1731564974588/Put/seqid=0 2024-11-14T06:16:15,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741843_1019 (size=14341) 2024-11-14T06:16:15,045 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/95788cae5bb34c848cfa2742fc1a674f 2024-11-14T06:16:15,076 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/65d479ab5cde4fafb53e84f46d8adb90 is 50, key is test_row_0/B:col10/1731564974588/Put/seqid=0 2024-11-14T06:16:15,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:15,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565035058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:15,088 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:15,088 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:15,088 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:15,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:15,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565035062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:15,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:15,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:15,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:15,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:15,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565035069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:15,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:15,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:15,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:15,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:15,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565035075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:15,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:15,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:15,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:15,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:15,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565035082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:15,102 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:15,102 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:15,103 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:15,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741844_1020 (size=12001) 2024-11-14T06:16:15,128 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/65d479ab5cde4fafb53e84f46d8adb90 2024-11-14T06:16:15,163 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/a0ebc431ebb34176afa2c13ad4d214a7 is 50, key is test_row_0/C:col10/1731564974588/Put/seqid=0 2024-11-14T06:16:15,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:15,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565035198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:15,203 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:15,203 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:15,203 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:15,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741845_1021 (size=12001) 2024-11-14T06:16:15,213 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/a0ebc431ebb34176afa2c13ad4d214a7 2024-11-14T06:16:15,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:15,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565035209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:15,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:15,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:15,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:15,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:15,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565035209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:15,217 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:15,217 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:15,217 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:15,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:15,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565035210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:15,222 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:15,223 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:15,223 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:15,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:15,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565035209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:15,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:15,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:15,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:15,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/95788cae5bb34c848cfa2742fc1a674f as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/95788cae5bb34c848cfa2742fc1a674f 2024-11-14T06:16:15,258 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/95788cae5bb34c848cfa2742fc1a674f, entries=200, sequenceid=48, filesize=14.0 K 2024-11-14T06:16:15,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/65d479ab5cde4fafb53e84f46d8adb90 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/65d479ab5cde4fafb53e84f46d8adb90 2024-11-14T06:16:15,274 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/65d479ab5cde4fafb53e84f46d8adb90, entries=150, sequenceid=48, filesize=11.7 K 2024-11-14T06:16:15,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/a0ebc431ebb34176afa2c13ad4d214a7 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/a0ebc431ebb34176afa2c13ad4d214a7 2024-11-14T06:16:15,298 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/a0ebc431ebb34176afa2c13ad4d214a7, entries=150, sequenceid=48, filesize=11.7 K 2024-11-14T06:16:15,300 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for d7851abd86d644f97c54179c322b1c1e in 342ms, sequenceid=48, compaction requested=true 2024-11-14T06:16:15,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d7851abd86d644f97c54179c322b1c1e: 2024-11-14T06:16:15,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7851abd86d644f97c54179c322b1c1e:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T06:16:15,329 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:16:15,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:16:15,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7851abd86d644f97c54179c322b1c1e:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T06:16:15,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:16:15,330 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:16:15,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7851abd86d644f97c54179c322b1c1e:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T06:16:15,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:16:15,333 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:16:15,335 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): d7851abd86d644f97c54179c322b1c1e/B is initiating minor compaction (all files) 2024-11-14T06:16:15,336 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7851abd86d644f97c54179c322b1c1e/B in TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:15,337 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/1f563e6300eb4128b97d7b2e5212cc22, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/20c53261f4ef42a2b16bc9f1f6198b4b, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/65d479ab5cde4fafb53e84f46d8adb90] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp, totalSize=35.2 K 2024-11-14T06:16:15,340 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f563e6300eb4128b97d7b2e5212cc22, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731564972012 2024-11-14T06:16:15,341 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 20c53261f4ef42a2b16bc9f1f6198b4b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731564972219 2024-11-14T06:16:15,343 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 65d479ab5cde4fafb53e84f46d8adb90, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731564974547 2024-11-14T06:16:15,345 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:16:15,345 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): d7851abd86d644f97c54179c322b1c1e/A is initiating minor compaction (all files) 2024-11-14T06:16:15,345 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7851abd86d644f97c54179c322b1c1e/A in TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:15,346 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/af35530d29884ab99f3e8671e990cf3e, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/3f3f93eafcd94ee9b649a4b323167c3c, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/95788cae5bb34c848cfa2742fc1a674f] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp, totalSize=37.4 K 2024-11-14T06:16:15,348 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting af35530d29884ab99f3e8671e990cf3e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731564972012 2024-11-14T06:16:15,351 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f3f93eafcd94ee9b649a4b323167c3c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731564972219 2024-11-14T06:16:15,352 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95788cae5bb34c848cfa2742fc1a674f, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731564974547 2024-11-14T06:16:15,412 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7851abd86d644f97c54179c322b1c1e#B#compaction#9 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:16:15,416 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7851abd86d644f97c54179c322b1c1e#A#compaction#10 average throughput is 0.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:16:15,417 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/0c834b1792724622a21cb2148c1bf1bf is 50, key is test_row_0/A:col10/1731564974588/Put/seqid=0 2024-11-14T06:16:15,419 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/a9081c74a881440c8ab00f9fcb0a7fff is 50, key is test_row_0/B:col10/1731564974588/Put/seqid=0 2024-11-14T06:16:15,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on d7851abd86d644f97c54179c322b1c1e 2024-11-14T06:16:15,421 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d7851abd86d644f97c54179c322b1c1e 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-14T06:16:15,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=A 2024-11-14T06:16:15,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:15,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=B 2024-11-14T06:16:15,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:15,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=C 2024-11-14T06:16:15,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:15,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741846_1022 (size=12104) 2024-11-14T06:16:15,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741847_1023 (size=12104) 2024-11-14T06:16:15,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/16408aa6ee244b21bbc620db4f11ddd4 is 50, key is test_row_0/A:col10/1731564975044/Put/seqid=0 2024-11-14T06:16:15,488 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/0c834b1792724622a21cb2148c1bf1bf as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/0c834b1792724622a21cb2148c1bf1bf 2024-11-14T06:16:15,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:15,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565035479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:15,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:15,506 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:15,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:15,506 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:15,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565035480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:15,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:15,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:15,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:15,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:15,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:15,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565035490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:15,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565035484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:15,508 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:15,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:15,508 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:15,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:15,508 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:15,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:15,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:15,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565035505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:15,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:15,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:15,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:15,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741848_1024 (size=14341) 2024-11-14T06:16:15,525 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/16408aa6ee244b21bbc620db4f11ddd4 2024-11-14T06:16:15,539 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-14T06:16:15,539 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-14T06:16:15,542 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T06:16:15,542 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-14T06:16:15,543 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-14T06:16:15,543 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-14T06:16:15,545 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-14T06:16:15,545 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-14T06:16:15,573 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/99fa4625cae249a2bb0dc0319d3e738a is 50, key is test_row_0/B:col10/1731564975044/Put/seqid=0 2024-11-14T06:16:15,591 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d7851abd86d644f97c54179c322b1c1e/A of d7851abd86d644f97c54179c322b1c1e into 0c834b1792724622a21cb2148c1bf1bf(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:16:15,592 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7851abd86d644f97c54179c322b1c1e: 2024-11-14T06:16:15,595 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., storeName=d7851abd86d644f97c54179c322b1c1e/A, priority=13, startTime=1731564975302; duration=0sec 2024-11-14T06:16:15,596 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:16:15,596 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7851abd86d644f97c54179c322b1c1e:A 2024-11-14T06:16:15,596 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:16:15,609 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:16:15,610 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): d7851abd86d644f97c54179c322b1c1e/C is initiating minor compaction (all files) 2024-11-14T06:16:15,610 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7851abd86d644f97c54179c322b1c1e/C in TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:15,610 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/0a933fe3e7754ed994c3e0ff2d4e1b33, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/16213f65774845a0b74d3d15ea6ae263, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/a0ebc431ebb34176afa2c13ad4d214a7] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp, totalSize=35.2 K 2024-11-14T06:16:15,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:15,612 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a933fe3e7754ed994c3e0ff2d4e1b33, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731564972012 2024-11-14T06:16:15,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565035608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:15,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:15,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:15,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:15,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:15,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565035618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:15,633 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:15,633 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:15,633 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:15,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:15,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565035618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:15,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:15,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:15,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:15,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:15,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565035619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:15,637 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16213f65774845a0b74d3d15ea6ae263, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731564972219 2024-11-14T06:16:15,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741849_1025 (size=12001) 2024-11-14T06:16:15,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:15,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:15,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:15,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:15,638 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0ebc431ebb34176afa2c13ad4d214a7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731564974547 2024-11-14T06:16:15,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565035619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:15,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:15,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:15,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:15,723 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7851abd86d644f97c54179c322b1c1e#C#compaction#13 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:16:15,724 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/5a5b9008a5aa41f09b69efab83710373 is 50, key is test_row_0/C:col10/1731564974588/Put/seqid=0 2024-11-14T06:16:15,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741850_1026 (size=12104) 2024-11-14T06:16:15,825 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/5a5b9008a5aa41f09b69efab83710373 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/5a5b9008a5aa41f09b69efab83710373 2024-11-14T06:16:15,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:15,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565035833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:15,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:15,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:15,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:15,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:15,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:15,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565035839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:15,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:15,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:15,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:15,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:15,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565035865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:15,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:15,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:15,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:15,885 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d7851abd86d644f97c54179c322b1c1e/C of d7851abd86d644f97c54179c322b1c1e into 5a5b9008a5aa41f09b69efab83710373(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:16:15,886 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7851abd86d644f97c54179c322b1c1e: 2024-11-14T06:16:15,886 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., storeName=d7851abd86d644f97c54179c322b1c1e/C, priority=13, startTime=1731564975330; duration=0sec 2024-11-14T06:16:15,886 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:16:15,886 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7851abd86d644f97c54179c322b1c1e:C 2024-11-14T06:16:15,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565035864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:15,889 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:15,889 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:15,889 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:15,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:15,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565035867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:15,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:15,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:15,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:15,924 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/a9081c74a881440c8ab00f9fcb0a7fff as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/a9081c74a881440c8ab00f9fcb0a7fff 2024-11-14T06:16:15,990 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d7851abd86d644f97c54179c322b1c1e/B of d7851abd86d644f97c54179c322b1c1e into a9081c74a881440c8ab00f9fcb0a7fff(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:16:15,990 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7851abd86d644f97c54179c322b1c1e: 2024-11-14T06:16:15,990 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., storeName=d7851abd86d644f97c54179c322b1c1e/B, priority=13, startTime=1731564975330; duration=0sec 2024-11-14T06:16:15,990 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:16:15,990 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7851abd86d644f97c54179c322b1c1e:B 2024-11-14T06:16:16,033 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/99fa4625cae249a2bb0dc0319d3e738a 2024-11-14T06:16:16,093 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/f0d404a8188b43689d6223f3cf7db3ff is 50, key is test_row_0/C:col10/1731564975044/Put/seqid=0 2024-11-14T06:16:16,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T06:16:16,133 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:16:16,133 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T06:16:16,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-14T06:16:16,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T06:16:16,140 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T06:16:16,142 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T06:16:16,142 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T06:16:16,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741851_1027 (size=12001) 2024-11-14T06:16:16,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:16,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565036179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:16,183 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:16,183 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:16,183 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:16,185 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/f0d404a8188b43689d6223f3cf7db3ff 2024-11-14T06:16:16,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:16,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565036188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:16,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:16,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:16,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:16,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:16,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565036188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:16,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/16408aa6ee244b21bbc620db4f11ddd4 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/16408aa6ee244b21bbc620db4f11ddd4 2024-11-14T06:16:16,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:16,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:16,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:16,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:16,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565036197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:16,214 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:16,215 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:16,215 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:16,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:16,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565036198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:16,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:16,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:16,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:16,232 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/16408aa6ee244b21bbc620db4f11ddd4, entries=200, sequenceid=66, filesize=14.0 K 2024-11-14T06:16:16,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/99fa4625cae249a2bb0dc0319d3e738a as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/99fa4625cae249a2bb0dc0319d3e738a 2024-11-14T06:16:16,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T06:16:16,257 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/99fa4625cae249a2bb0dc0319d3e738a, entries=150, sequenceid=66, filesize=11.7 K 2024-11-14T06:16:16,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/f0d404a8188b43689d6223f3cf7db3ff as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/f0d404a8188b43689d6223f3cf7db3ff 2024-11-14T06:16:16,277 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/f0d404a8188b43689d6223f3cf7db3ff, entries=150, sequenceid=66, filesize=11.7 K 2024-11-14T06:16:16,279 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for d7851abd86d644f97c54179c322b1c1e in 858ms, sequenceid=66, compaction requested=false 2024-11-14T06:16:16,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d7851abd86d644f97c54179c322b1c1e: 2024-11-14T06:16:16,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T06:16:16,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:16,306 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing d7851abd86d644f97c54179c322b1c1e 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-14T06:16:16,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=A 2024-11-14T06:16:16,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:16,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=B 2024-11-14T06:16:16,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:16,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=C 2024-11-14T06:16:16,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:16,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/076c9070dc7b4233a87552e384aa22e9 is 50, key is test_row_0/A:col10/1731564975497/Put/seqid=0 2024-11-14T06:16:16,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741852_1028 (size=12001) 2024-11-14T06:16:16,414 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/076c9070dc7b4233a87552e384aa22e9 2024-11-14T06:16:16,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/01e3ddeefabe44b9b0bb779bf4ee3d4e is 50, key is test_row_0/B:col10/1731564975497/Put/seqid=0 2024-11-14T06:16:16,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T06:16:16,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741853_1029 (size=12001) 2024-11-14T06:16:16,479 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/01e3ddeefabe44b9b0bb779bf4ee3d4e 2024-11-14T06:16:16,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/d71c253f64d14777b77f41b83ebceabc is 50, key is test_row_0/C:col10/1731564975497/Put/seqid=0 2024-11-14T06:16:16,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741854_1030 (size=12001) 2024-11-14T06:16:16,562 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/d71c253f64d14777b77f41b83ebceabc 2024-11-14T06:16:16,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/076c9070dc7b4233a87552e384aa22e9 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/076c9070dc7b4233a87552e384aa22e9 2024-11-14T06:16:16,604 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/076c9070dc7b4233a87552e384aa22e9, entries=150, sequenceid=83, filesize=11.7 K 2024-11-14T06:16:16,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/01e3ddeefabe44b9b0bb779bf4ee3d4e as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/01e3ddeefabe44b9b0bb779bf4ee3d4e 2024-11-14T06:16:16,636 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/01e3ddeefabe44b9b0bb779bf4ee3d4e, entries=150, sequenceid=83, filesize=11.7 K 2024-11-14T06:16:16,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/d71c253f64d14777b77f41b83ebceabc as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/d71c253f64d14777b77f41b83ebceabc 2024-11-14T06:16:16,653 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/d71c253f64d14777b77f41b83ebceabc, entries=150, sequenceid=83, filesize=11.7 K 2024-11-14T06:16:16,655 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=0 B/0 for d7851abd86d644f97c54179c322b1c1e in 349ms, sequenceid=83, compaction requested=true 2024-11-14T06:16:16,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for d7851abd86d644f97c54179c322b1c1e: 2024-11-14T06:16:16,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:16,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-14T06:16:16,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-14T06:16:16,663 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-14T06:16:16,663 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 516 msec 2024-11-14T06:16:16,669 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 531 msec 2024-11-14T06:16:16,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on d7851abd86d644f97c54179c322b1c1e 2024-11-14T06:16:16,754 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d7851abd86d644f97c54179c322b1c1e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T06:16:16,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=A 2024-11-14T06:16:16,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:16,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=B 2024-11-14T06:16:16,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:16,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=C 2024-11-14T06:16:16,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:16,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/11d7d3ab91a6490b9e6c598cb8fc713c is 50, key is test_row_0/A:col10/1731564976746/Put/seqid=0 2024-11-14T06:16:16,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T06:16:16,788 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:16:16,789 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T06:16:16,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-14T06:16:16,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T06:16:16,795 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T06:16:16,798 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T06:16:16,798 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T06:16:16,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741855_1031 (size=19021) 2024-11-14T06:16:16,829 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/11d7d3ab91a6490b9e6c598cb8fc713c 2024-11-14T06:16:16,873 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/73ce2a6aa2ad4d519ba9fea5d618943d is 50, key is test_row_0/B:col10/1731564976746/Put/seqid=0 2024-11-14T06:16:16,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T06:16:16,942 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731564976855, max timestamp=1731564976892Num uniques 859; 2024-11-14T06:16:16,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T06:16:16,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T06:16:16,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:16,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. as already flushing 2024-11-14T06:16:16,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:16,955 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:16,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:16,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:16,961 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731564976855, max timestamp=1731564976889Num uniques 859; 2024-11-14T06:16:16,961 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T06:16:16,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741856_1032 (size=12001) 2024-11-14T06:16:16,986 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/73ce2a6aa2ad4d519ba9fea5d618943d 2024-11-14T06:16:17,010 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731564976855, max timestamp=1731564976892Num uniques 859; 2024-11-14T06:16:17,010 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-14T06:16:17,020 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/1c9508b2673445e58e886f51e9712b78 is 50, key is test_row_0/C:col10/1731564976746/Put/seqid=0 2024-11-14T06:16:17,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:17,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565037048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:17,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:17,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:17,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:17,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741857_1033 (size=12001) 2024-11-14T06:16:17,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T06:16:17,110 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/1c9508b2673445e58e886f51e9712b78 2024-11-14T06:16:17,116 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731564977029, max timestamp=1731564977090Num uniques 1018; 2024-11-14T06:16:17,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T06:16:17,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:17,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. as already flushing 2024-11-14T06:16:17,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:17,123 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:17,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:17,124 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=14558, suffix heap size=133840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=318 2024-11-14T06:16:17,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/11d7d3ab91a6490b9e6c598cb8fc713c as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/11d7d3ab91a6490b9e6c598cb8fc713c 2024-11-14T06:16:17,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:17,131 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731564977029, max timestamp=1731564977090Num uniques 1018; 2024-11-14T06:16:17,131 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=14558, suffix heap size=133840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=318 2024-11-14T06:16:17,139 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/11d7d3ab91a6490b9e6c598cb8fc713c, entries=300, sequenceid=93, filesize=18.6 K 2024-11-14T06:16:17,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/73ce2a6aa2ad4d519ba9fea5d618943d as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/73ce2a6aa2ad4d519ba9fea5d618943d 2024-11-14T06:16:17,166 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/73ce2a6aa2ad4d519ba9fea5d618943d, entries=150, sequenceid=93, filesize=11.7 K 2024-11-14T06:16:17,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/1c9508b2673445e58e886f51e9712b78 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/1c9508b2673445e58e886f51e9712b78 2024-11-14T06:16:17,172 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731564977035, max timestamp=1731564977090Num uniques 1009; 2024-11-14T06:16:17,173 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=6870, suffix heap size=132904,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=150 2024-11-14T06:16:17,199 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/1c9508b2673445e58e886f51e9712b78, entries=150, sequenceid=93, filesize=11.7 K 2024-11-14T06:16:17,209 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=86.42 KB/88492 for d7851abd86d644f97c54179c322b1c1e in 455ms, sequenceid=93, compaction requested=true 2024-11-14T06:16:17,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d7851abd86d644f97c54179c322b1c1e: 2024-11-14T06:16:17,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7851abd86d644f97c54179c322b1c1e:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T06:16:17,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:16:17,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7851abd86d644f97c54179c322b1c1e:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T06:16:17,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T06:16:17,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7851abd86d644f97c54179c322b1c1e:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T06:16:17,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-14T06:16:17,211 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T06:16:17,211 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T06:16:17,221 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57467 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T06:16:17,221 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): d7851abd86d644f97c54179c322b1c1e/A is initiating minor compaction (all files) 2024-11-14T06:16:17,222 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7851abd86d644f97c54179c322b1c1e/A in TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:17,222 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/0c834b1792724622a21cb2148c1bf1bf, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/16408aa6ee244b21bbc620db4f11ddd4, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/076c9070dc7b4233a87552e384aa22e9, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/11d7d3ab91a6490b9e6c598cb8fc713c] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp, totalSize=56.1 K 2024-11-14T06:16:17,222 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T06:16:17,223 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): d7851abd86d644f97c54179c322b1c1e/C is initiating minor compaction (all files) 2024-11-14T06:16:17,223 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7851abd86d644f97c54179c322b1c1e/C in TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:17,223 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/5a5b9008a5aa41f09b69efab83710373, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/f0d404a8188b43689d6223f3cf7db3ff, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/d71c253f64d14777b77f41b83ebceabc, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/1c9508b2673445e58e886f51e9712b78] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp, totalSize=47.0 K 2024-11-14T06:16:17,225 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a5b9008a5aa41f09b69efab83710373, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731564974547 2024-11-14T06:16:17,226 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c834b1792724622a21cb2148c1bf1bf, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731564974547 2024-11-14T06:16:17,228 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16408aa6ee244b21bbc620db4f11ddd4, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731564975044 2024-11-14T06:16:17,229 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting f0d404a8188b43689d6223f3cf7db3ff, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731564975044 2024-11-14T06:16:17,230 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting d71c253f64d14777b77f41b83ebceabc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731564975471 2024-11-14T06:16:17,231 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 076c9070dc7b4233a87552e384aa22e9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731564975471 2024-11-14T06:16:17,232 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11d7d3ab91a6490b9e6c598cb8fc713c, keycount=300, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731564976707 2024-11-14T06:16:17,233 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c9508b2673445e58e886f51e9712b78, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731564976740 2024-11-14T06:16:17,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on d7851abd86d644f97c54179c322b1c1e 2024-11-14T06:16:17,238 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d7851abd86d644f97c54179c322b1c1e 3/3 column families, dataSize=93.13 KB heapSize=253.41 KB 2024-11-14T06:16:17,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=A 2024-11-14T06:16:17,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:16:17,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=B 2024-11-14T06:16:17,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:16:17,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=C 2024-11-14T06:16:17,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:16:17,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/dac0f0fc524f47bbbce528f6d564924b is 50, key is test_row_0/A:col10/1731564977134/Put/seqid=0 2024-11-14T06:16:17,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T06:16:17,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:17,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. as already flushing 2024-11-14T06:16:17,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:17,280 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:17,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:17,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:17,298 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7851abd86d644f97c54179c322b1c1e#A#compaction#22 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:16:17,301 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/170d5fcbeed741cdb7fb3e71b8082d87 is 50, key is test_row_0/A:col10/1731564976746/Put/seqid=0 2024-11-14T06:16:17,333 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7851abd86d644f97c54179c322b1c1e#C#compaction#23 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:16:17,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:17,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565037307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:17,334 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/cf9daf43a6984455b4a9bb3e71681e7a is 50, key is test_row_0/C:col10/1731564976746/Put/seqid=0 2024-11-14T06:16:17,335 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:17,335 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:17,335 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:17,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:17,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565037327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:17,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:17,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:17,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:17,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:17,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565037330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:17,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:17,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:17,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:17,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:17,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565037334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:17,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:17,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:17,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:17,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:17,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565037334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:17,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:17,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:17,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:17,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741858_1034 (size=14541) 2024-11-14T06:16:17,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T06:16:17,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741859_1035 (size=12241) 2024-11-14T06:16:17,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T06:16:17,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:17,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. as already flushing 2024-11-14T06:16:17,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:17,436 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:17,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:17,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:17,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741860_1036 (size=12241) 2024-11-14T06:16:17,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:17,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565037438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:17,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:17,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:17,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:17,451 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/170d5fcbeed741cdb7fb3e71b8082d87 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/170d5fcbeed741cdb7fb3e71b8082d87 2024-11-14T06:16:17,461 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/cf9daf43a6984455b4a9bb3e71681e7a as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/cf9daf43a6984455b4a9bb3e71681e7a 2024-11-14T06:16:17,465 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d7851abd86d644f97c54179c322b1c1e/A of d7851abd86d644f97c54179c322b1c1e into 170d5fcbeed741cdb7fb3e71b8082d87(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:16:17,465 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7851abd86d644f97c54179c322b1c1e: 2024-11-14T06:16:17,465 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., storeName=d7851abd86d644f97c54179c322b1c1e/A, priority=12, startTime=1731564977210; duration=0sec 2024-11-14T06:16:17,466 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:16:17,466 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7851abd86d644f97c54179c322b1c1e:A 2024-11-14T06:16:17,466 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T06:16:17,471 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T06:16:17,471 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): d7851abd86d644f97c54179c322b1c1e/B is initiating minor compaction (all files) 2024-11-14T06:16:17,472 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7851abd86d644f97c54179c322b1c1e/B in TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:17,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:17,472 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/a9081c74a881440c8ab00f9fcb0a7fff, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/99fa4625cae249a2bb0dc0319d3e738a, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/01e3ddeefabe44b9b0bb779bf4ee3d4e, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/73ce2a6aa2ad4d519ba9fea5d618943d] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp, totalSize=47.0 K 2024-11-14T06:16:17,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565037459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:17,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:17,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:17,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:17,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:17,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565037459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:17,474 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9081c74a881440c8ab00f9fcb0a7fff, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731564974547 2024-11-14T06:16:17,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:17,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:17,475 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99fa4625cae249a2bb0dc0319d3e738a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731564975044 2024-11-14T06:16:17,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:17,476 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01e3ddeefabe44b9b0bb779bf4ee3d4e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731564975471 2024-11-14T06:16:17,477 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d7851abd86d644f97c54179c322b1c1e/C of d7851abd86d644f97c54179c322b1c1e into cf9daf43a6984455b4a9bb3e71681e7a(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:16:17,477 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7851abd86d644f97c54179c322b1c1e: 2024-11-14T06:16:17,477 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., storeName=d7851abd86d644f97c54179c322b1c1e/C, priority=12, startTime=1731564977211; duration=0sec 2024-11-14T06:16:17,477 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73ce2a6aa2ad4d519ba9fea5d618943d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731564976740 2024-11-14T06:16:17,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:17,478 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:16:17,478 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7851abd86d644f97c54179c322b1c1e:C 2024-11-14T06:16:17,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565037459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:17,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:17,480 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:17,480 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:17,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:17,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565037467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:17,485 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:17,485 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:17,485 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:17,509 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7851abd86d644f97c54179c322b1c1e#B#compaction#24 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:16:17,509 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/8f82a4635d8b40b59c53584415923c62 is 50, key is test_row_0/B:col10/1731564976746/Put/seqid=0 2024-11-14T06:16:17,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741861_1037 (size=12241) 2024-11-14T06:16:17,555 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/8f82a4635d8b40b59c53584415923c62 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/8f82a4635d8b40b59c53584415923c62 2024-11-14T06:16:17,568 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d7851abd86d644f97c54179c322b1c1e/B of d7851abd86d644f97c54179c322b1c1e into 8f82a4635d8b40b59c53584415923c62(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:16:17,568 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7851abd86d644f97c54179c322b1c1e: 2024-11-14T06:16:17,568 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., storeName=d7851abd86d644f97c54179c322b1c1e/B, priority=12, startTime=1731564977210; duration=0sec 2024-11-14T06:16:17,569 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:16:17,569 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7851abd86d644f97c54179c322b1c1e:B 2024-11-14T06:16:17,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T06:16:17,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:17,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. as already flushing 2024-11-14T06:16:17,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:17,595 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:17,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:17,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:17,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:17,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565037661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:17,674 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:17,675 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:17,675 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:17,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:17,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565037682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:17,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:17,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:17,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:17,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:17,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:17,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565037694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:17,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:17,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:17,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:17,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:17,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565037694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:17,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565037694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:17,717 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:17,717 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:17,717 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:17,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:17,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:17,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:17,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T06:16:17,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:17,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. as already flushing 2024-11-14T06:16:17,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:17,761 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:17,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:17,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:17,765 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/dac0f0fc524f47bbbce528f6d564924b 2024-11-14T06:16:17,806 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/1c5d2474929e48cbb5ccea57ebd46638 is 50, key is test_row_0/B:col10/1731564977134/Put/seqid=0 2024-11-14T06:16:17,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741862_1038 (size=12151) 2024-11-14T06:16:17,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T06:16:17,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:17,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. as already flushing 2024-11-14T06:16:17,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:17,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:17,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:17,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:17,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T06:16:17,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:18,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565037998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:18,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:18,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:18,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:18,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:18,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565038000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:18,004 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:18,004 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:18,004 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:18,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:18,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:18,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565038028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:18,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:18,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:18,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:18,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565038028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:18,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:18,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:18,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:18,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:18,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565038028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:18,051 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:18,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:18,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:18,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T06:16:18,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:18,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. as already flushing 2024-11-14T06:16:18,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:18,075 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:18,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:18,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:18,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T06:16:18,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:18,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. as already flushing 2024-11-14T06:16:18,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:18,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:18,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:18,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:18,272 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/1c5d2474929e48cbb5ccea57ebd46638 2024-11-14T06:16:18,303 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/f25b70a6e6224683b471f27faf7122f0 is 50, key is test_row_0/C:col10/1731564977134/Put/seqid=0 2024-11-14T06:16:18,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741863_1039 (size=12151) 2024-11-14T06:16:18,396 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T06:16:18,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:18,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. as already flushing 2024-11-14T06:16:18,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:18,397 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:18,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:18,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:18,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:18,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565038512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:18,524 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:18,524 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:18,524 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:18,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:18,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565038512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:18,531 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:18,531 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:18,531 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:18,555 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T06:16:18,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:18,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. as already flushing 2024-11-14T06:16:18,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:18,556 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:18,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:18,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:18,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:18,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565038559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:18,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:18,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:18,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:18,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:18,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565038559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:18,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565038559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:18,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:18,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:18,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:18,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:18,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:18,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:18,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:18,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T06:16:18,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:18,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. as already flushing 2024-11-14T06:16:18,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:18,713 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:18,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:18,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:18,748 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.04 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/f25b70a6e6224683b471f27faf7122f0 2024-11-14T06:16:18,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/dac0f0fc524f47bbbce528f6d564924b as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/dac0f0fc524f47bbbce528f6d564924b 2024-11-14T06:16:18,772 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/dac0f0fc524f47bbbce528f6d564924b, entries=200, sequenceid=139, filesize=14.2 K 2024-11-14T06:16:18,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/1c5d2474929e48cbb5ccea57ebd46638 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/1c5d2474929e48cbb5ccea57ebd46638 2024-11-14T06:16:18,787 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/1c5d2474929e48cbb5ccea57ebd46638, entries=150, sequenceid=139, filesize=11.9 K 2024-11-14T06:16:18,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/f25b70a6e6224683b471f27faf7122f0 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/f25b70a6e6224683b471f27faf7122f0 2024-11-14T06:16:18,802 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/f25b70a6e6224683b471f27faf7122f0, entries=150, sequenceid=139, filesize=11.9 K 2024-11-14T06:16:18,804 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.13 KB/95362, heapSize ~253.36 KB/259440, currentSize=87.22 KB/89310 for d7851abd86d644f97c54179c322b1c1e in 1566ms, sequenceid=139, compaction requested=false 2024-11-14T06:16:18,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d7851abd86d644f97c54179c322b1c1e: 2024-11-14T06:16:18,869 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T06:16:18,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:18,870 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing d7851abd86d644f97c54179c322b1c1e 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-14T06:16:18,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=A 2024-11-14T06:16:18,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:18,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=B 2024-11-14T06:16:18,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:18,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=C 2024-11-14T06:16:18,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:18,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/cedf4b535a064b74bff85cbe09ff2c56 is 50, key is test_row_0/A:col10/1731564977303/Put/seqid=0 2024-11-14T06:16:18,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741864_1040 (size=12151) 2024-11-14T06:16:18,927 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/cedf4b535a064b74bff85cbe09ff2c56 2024-11-14T06:16:18,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T06:16:18,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/d14dd429dbb14487a243d5f21a0dab56 is 50, key is test_row_0/B:col10/1731564977303/Put/seqid=0 2024-11-14T06:16:19,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741865_1041 (size=12151) 2024-11-14T06:16:19,070 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/d14dd429dbb14487a243d5f21a0dab56 2024-11-14T06:16:19,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/85e3a4dd2df7471dba782e9613e1cebb is 50, key is test_row_0/C:col10/1731564977303/Put/seqid=0 2024-11-14T06:16:19,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741866_1042 (size=12151) 2024-11-14T06:16:19,149 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/85e3a4dd2df7471dba782e9613e1cebb 2024-11-14T06:16:19,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/cedf4b535a064b74bff85cbe09ff2c56 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/cedf4b535a064b74bff85cbe09ff2c56 2024-11-14T06:16:19,173 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/cedf4b535a064b74bff85cbe09ff2c56, entries=150, sequenceid=158, filesize=11.9 K 2024-11-14T06:16:19,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/d14dd429dbb14487a243d5f21a0dab56 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/d14dd429dbb14487a243d5f21a0dab56 2024-11-14T06:16:19,187 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/d14dd429dbb14487a243d5f21a0dab56, entries=150, sequenceid=158, filesize=11.9 K 2024-11-14T06:16:19,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/85e3a4dd2df7471dba782e9613e1cebb as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/85e3a4dd2df7471dba782e9613e1cebb 2024-11-14T06:16:19,202 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/85e3a4dd2df7471dba782e9613e1cebb, entries=150, sequenceid=158, filesize=11.9 K 2024-11-14T06:16:19,205 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=0 B/0 for d7851abd86d644f97c54179c322b1c1e in 335ms, sequenceid=158, compaction requested=true 2024-11-14T06:16:19,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for d7851abd86d644f97c54179c322b1c1e: 2024-11-14T06:16:19,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:19,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-14T06:16:19,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-14T06:16:19,212 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-14T06:16:19,213 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4100 sec 2024-11-14T06:16:19,217 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 2.4250 sec 2024-11-14T06:16:19,590 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d7851abd86d644f97c54179c322b1c1e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T06:16:19,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=A 2024-11-14T06:16:19,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:19,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=B 2024-11-14T06:16:19,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:19,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=C 2024-11-14T06:16:19,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:19,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on d7851abd86d644f97c54179c322b1c1e 2024-11-14T06:16:19,626 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/8e6d8ad5a2f94c649051d403cab83cd4 is 50, key is test_row_0/A:col10/1731564979587/Put/seqid=0 2024-11-14T06:16:19,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741867_1043 (size=19321) 2024-11-14T06:16:19,688 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/8e6d8ad5a2f94c649051d403cab83cd4 2024-11-14T06:16:19,727 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731564979663, max timestamp=1731564979691Num uniques 859; 2024-11-14T06:16:19,727 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T06:16:19,731 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731564979663, max timestamp=1731564979691Num uniques 859; 2024-11-14T06:16:19,732 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-14T06:16:19,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/0d40018fb1564f6e9bd47d0eeca33170 is 50, key is test_row_0/B:col10/1731564979587/Put/seqid=0 2024-11-14T06:16:19,759 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731564979663, max timestamp=1731564979710Num uniques 859; 2024-11-14T06:16:19,759 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-14T06:16:19,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741868_1044 (size=12151) 2024-11-14T06:16:19,803 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/0d40018fb1564f6e9bd47d0eeca33170 2024-11-14T06:16:19,838 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/0ce19d0e90d54864a8b7a9848093b3a0 is 50, key is test_row_0/C:col10/1731564979587/Put/seqid=0 2024-11-14T06:16:19,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:19,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565039848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:19,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741869_1045 (size=12151) 2024-11-14T06:16:19,900 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731564979838, max timestamp=1731564979876Num uniques 1018; 2024-11-14T06:16:19,901 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=6870, suffix heap size=133840,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=150 2024-11-14T06:16:19,904 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/0ce19d0e90d54864a8b7a9848093b3a0 2024-11-14T06:16:19,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/8e6d8ad5a2f94c649051d403cab83cd4 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/8e6d8ad5a2f94c649051d403cab83cd4 2024-11-14T06:16:19,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:19,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:19,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:19,885 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731564979842, max timestamp=1731564979876Num uniques 1009; 2024-11-14T06:16:19,922 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=6870, suffix heap size=132904,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=150 2024-11-14T06:16:19,922 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731564979842, max timestamp=1731564979876Num uniques 1009; 2024-11-14T06:16:19,922 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=6870, suffix heap size=132904,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=150 2024-11-14T06:16:19,941 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/8e6d8ad5a2f94c649051d403cab83cd4, entries=300, sequenceid=168, filesize=18.9 K 2024-11-14T06:16:19,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/0d40018fb1564f6e9bd47d0eeca33170 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/0d40018fb1564f6e9bd47d0eeca33170 2024-11-14T06:16:19,958 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/0d40018fb1564f6e9bd47d0eeca33170, entries=150, sequenceid=168, filesize=11.9 K 2024-11-14T06:16:19,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/0ce19d0e90d54864a8b7a9848093b3a0 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/0ce19d0e90d54864a8b7a9848093b3a0 2024-11-14T06:16:19,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/0ce19d0e90d54864a8b7a9848093b3a0, entries=150, sequenceid=168, filesize=11.9 K 2024-11-14T06:16:19,978 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=118.37 KB/121206 for d7851abd86d644f97c54179c322b1c1e in 388ms, sequenceid=168, compaction requested=true 2024-11-14T06:16:19,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d7851abd86d644f97c54179c322b1c1e: 2024-11-14T06:16:19,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7851abd86d644f97c54179c322b1c1e:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T06:16:19,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:16:19,979 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T06:16:19,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7851abd86d644f97c54179c322b1c1e:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T06:16:19,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:16:19,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7851abd86d644f97c54179c322b1c1e:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T06:16:19,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T06:16:19,980 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T06:16:19,981 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 58254 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T06:16:19,981 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): d7851abd86d644f97c54179c322b1c1e/A is initiating minor compaction (all files) 2024-11-14T06:16:19,982 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7851abd86d644f97c54179c322b1c1e/A in TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:19,982 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/170d5fcbeed741cdb7fb3e71b8082d87, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/dac0f0fc524f47bbbce528f6d564924b, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/cedf4b535a064b74bff85cbe09ff2c56, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/8e6d8ad5a2f94c649051d403cab83cd4] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp, totalSize=56.9 K 2024-11-14T06:16:19,983 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 170d5fcbeed741cdb7fb3e71b8082d87, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731564976740 2024-11-14T06:16:19,984 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48694 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T06:16:19,984 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): d7851abd86d644f97c54179c322b1c1e/B is initiating minor compaction (all files) 2024-11-14T06:16:19,984 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7851abd86d644f97c54179c322b1c1e/B in TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:19,984 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/8f82a4635d8b40b59c53584415923c62, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/1c5d2474929e48cbb5ccea57ebd46638, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/d14dd429dbb14487a243d5f21a0dab56, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/0d40018fb1564f6e9bd47d0eeca33170] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp, totalSize=47.6 K 2024-11-14T06:16:19,985 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting dac0f0fc524f47bbbce528f6d564924b, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1731564977134 2024-11-14T06:16:19,985 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f82a4635d8b40b59c53584415923c62, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731564976740 2024-11-14T06:16:19,986 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting cedf4b535a064b74bff85cbe09ff2c56, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731564977303 2024-11-14T06:16:19,986 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c5d2474929e48cbb5ccea57ebd46638, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1731564977134 2024-11-14T06:16:19,987 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e6d8ad5a2f94c649051d403cab83cd4, keycount=300, bloomtype=ROW, size=18.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731564979543 2024-11-14T06:16:19,987 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting d14dd429dbb14487a243d5f21a0dab56, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731564977303 2024-11-14T06:16:19,989 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d40018fb1564f6e9bd47d0eeca33170, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731564979577 2024-11-14T06:16:19,991 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d7851abd86d644f97c54179c322b1c1e 3/3 column families, dataSize=125.07 KB heapSize=358.73 KB 2024-11-14T06:16:19,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=A 2024-11-14T06:16:19,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:16:19,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=B 2024-11-14T06:16:19,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:16:19,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=C 2024-11-14T06:16:19,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:16:19,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on d7851abd86d644f97c54179c322b1c1e 2024-11-14T06:16:20,008 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/dbead1a3634346b29b1f95558d23fa76 is 50, key is test_row_0/A:col10/1731564979966/Put/seqid=0 2024-11-14T06:16:20,020 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7851abd86d644f97c54179c322b1c1e#B#compaction#34 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:16:20,021 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/87d78efde7e949ea8a5a3dc793592d3c is 50, key is test_row_0/B:col10/1731564979587/Put/seqid=0 2024-11-14T06:16:20,031 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7851abd86d644f97c54179c322b1c1e#A#compaction#35 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:16:20,033 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/bbf5047c48354d9d99f9cf697cd7335f is 50, key is test_row_0/A:col10/1731564979587/Put/seqid=0 2024-11-14T06:16:20,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:20,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565040043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:20,096 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:20,096 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:20,096 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:20,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:20,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565040057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:20,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:20,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565040067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:20,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:20,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:20,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:20,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:20,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565040075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:20,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:20,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:20,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:20,113 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:20,113 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:20,113 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:20,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:20,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741870_1046 (size=12527) 2024-11-14T06:16:20,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565040094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:20,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:20,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:20,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:20,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741871_1047 (size=19321) 2024-11-14T06:16:20,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741872_1048 (size=12527) 2024-11-14T06:16:20,170 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/bbf5047c48354d9d99f9cf697cd7335f as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/bbf5047c48354d9d99f9cf697cd7335f 2024-11-14T06:16:20,183 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d7851abd86d644f97c54179c322b1c1e/A of d7851abd86d644f97c54179c322b1c1e into bbf5047c48354d9d99f9cf697cd7335f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:16:20,184 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7851abd86d644f97c54179c322b1c1e: 2024-11-14T06:16:20,184 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., storeName=d7851abd86d644f97c54179c322b1c1e/A, priority=12, startTime=1731564979978; duration=0sec 2024-11-14T06:16:20,184 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:16:20,184 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7851abd86d644f97c54179c322b1c1e:A 2024-11-14T06:16:20,184 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T06:16:20,187 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48694 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T06:16:20,187 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): d7851abd86d644f97c54179c322b1c1e/C is initiating minor compaction (all files) 2024-11-14T06:16:20,187 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7851abd86d644f97c54179c322b1c1e/C in TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:20,187 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/cf9daf43a6984455b4a9bb3e71681e7a, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/f25b70a6e6224683b471f27faf7122f0, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/85e3a4dd2df7471dba782e9613e1cebb, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/0ce19d0e90d54864a8b7a9848093b3a0] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp, totalSize=47.6 K 2024-11-14T06:16:20,188 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting cf9daf43a6984455b4a9bb3e71681e7a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731564976740 2024-11-14T06:16:20,190 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting f25b70a6e6224683b471f27faf7122f0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1731564977134 2024-11-14T06:16:20,192 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 85e3a4dd2df7471dba782e9613e1cebb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731564977303 2024-11-14T06:16:20,193 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ce19d0e90d54864a8b7a9848093b3a0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731564979577 2024-11-14T06:16:20,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:20,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565040207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:20,214 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:20,214 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:20,214 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:20,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:20,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565040210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:20,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:20,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:20,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:20,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:20,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565040218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:20,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:20,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:20,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:20,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:20,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565040218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:20,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:20,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:20,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:20,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:20,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565040218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:20,230 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7851abd86d644f97c54179c322b1c1e#C#compaction#36 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:16:20,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:20,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:20,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:20,230 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/bcaca51cb497417fa62e2168fa7ceea4 is 50, key is test_row_0/C:col10/1731564979587/Put/seqid=0 2024-11-14T06:16:20,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741873_1049 (size=12527) 2024-11-14T06:16:20,290 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/bcaca51cb497417fa62e2168fa7ceea4 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/bcaca51cb497417fa62e2168fa7ceea4 2024-11-14T06:16:20,299 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d7851abd86d644f97c54179c322b1c1e/C of d7851abd86d644f97c54179c322b1c1e into bcaca51cb497417fa62e2168fa7ceea4(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:16:20,300 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7851abd86d644f97c54179c322b1c1e: 2024-11-14T06:16:20,300 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., storeName=d7851abd86d644f97c54179c322b1c1e/C, priority=12, startTime=1731564979979; duration=0sec 2024-11-14T06:16:20,300 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:16:20,300 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7851abd86d644f97c54179c322b1c1e:C 2024-11-14T06:16:20,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:20,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565040418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:20,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:20,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:20,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:20,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:20,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565040427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:20,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:20,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:20,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:20,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:20,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565040437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:20,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:20,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:20,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:20,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:20,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565040438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:20,441 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:20,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:20,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:20,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:20,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565040438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:20,443 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:20,443 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:20,443 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:20,524 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/87d78efde7e949ea8a5a3dc793592d3c as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/87d78efde7e949ea8a5a3dc793592d3c 2024-11-14T06:16:20,528 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.69 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/dbead1a3634346b29b1f95558d23fa76 2024-11-14T06:16:20,540 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d7851abd86d644f97c54179c322b1c1e/B of d7851abd86d644f97c54179c322b1c1e into 87d78efde7e949ea8a5a3dc793592d3c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:16:20,540 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7851abd86d644f97c54179c322b1c1e: 2024-11-14T06:16:20,541 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., storeName=d7851abd86d644f97c54179c322b1c1e/B, priority=12, startTime=1731564979979; duration=0sec 2024-11-14T06:16:20,541 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:16:20,541 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7851abd86d644f97c54179c322b1c1e:B 2024-11-14T06:16:20,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/ebe2c26aa8e64e5daf006f8bbacc1fb2 is 50, key is test_row_0/B:col10/1731564979966/Put/seqid=0 2024-11-14T06:16:20,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741874_1050 (size=12151) 2024-11-14T06:16:20,605 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.69 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/ebe2c26aa8e64e5daf006f8bbacc1fb2 2024-11-14T06:16:20,645 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/0162a3e74f184f49a26f74983c6ce4b6 is 50, key is test_row_0/C:col10/1731564979966/Put/seqid=0 2024-11-14T06:16:20,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741875_1051 (size=12151) 2024-11-14T06:16:20,677 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.69 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/0162a3e74f184f49a26f74983c6ce4b6 2024-11-14T06:16:20,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/dbead1a3634346b29b1f95558d23fa76 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/dbead1a3634346b29b1f95558d23fa76 2024-11-14T06:16:20,705 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/dbead1a3634346b29b1f95558d23fa76, entries=300, sequenceid=221, filesize=18.9 K 2024-11-14T06:16:20,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/ebe2c26aa8e64e5daf006f8bbacc1fb2 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/ebe2c26aa8e64e5daf006f8bbacc1fb2 2024-11-14T06:16:20,717 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/ebe2c26aa8e64e5daf006f8bbacc1fb2, entries=150, sequenceid=221, filesize=11.9 K 2024-11-14T06:16:20,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/0162a3e74f184f49a26f74983c6ce4b6 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/0162a3e74f184f49a26f74983c6ce4b6 2024-11-14T06:16:20,738 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/0162a3e74f184f49a26f74983c6ce4b6, entries=150, sequenceid=221, filesize=11.9 K 2024-11-14T06:16:20,741 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~125.07 KB/128076, heapSize ~358.69 KB/367296, currentSize=53.67 KB/54960 for d7851abd86d644f97c54179c322b1c1e in 751ms, sequenceid=221, compaction requested=false 2024-11-14T06:16:20,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d7851abd86d644f97c54179c322b1c1e: 2024-11-14T06:16:20,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on d7851abd86d644f97c54179c322b1c1e 2024-11-14T06:16:20,763 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d7851abd86d644f97c54179c322b1c1e 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-14T06:16:20,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=A 2024-11-14T06:16:20,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:20,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=B 2024-11-14T06:16:20,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:20,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=C 2024-11-14T06:16:20,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:20,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/27afff0584af472995f9addfb8b48bc4 is 50, key is test_row_0/A:col10/1731564980034/Put/seqid=0 2024-11-14T06:16:20,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:20,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565040842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:20,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:20,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:20,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:20,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:20,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565040843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:20,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:20,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565040847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:20,854 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:20,855 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:20,855 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:20,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:20,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:20,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:20,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:20,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565040851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:20,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:20,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:20,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:20,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:20,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565040855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:20,860 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:20,860 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:20,860 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:20,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741876_1052 (size=12151) 2024-11-14T06:16:20,901 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/27afff0584af472995f9addfb8b48bc4 2024-11-14T06:16:20,913 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/d0dbf4cfdb434649a9e643ee7fc9d8f3 is 50, key is test_row_0/B:col10/1731564980034/Put/seqid=0 2024-11-14T06:16:20,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741877_1053 (size=12151) 2024-11-14T06:16:20,946 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/d0dbf4cfdb434649a9e643ee7fc9d8f3 2024-11-14T06:16:20,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T06:16:20,958 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:16:20,960 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T06:16:20,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:20,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565040960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:20,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:20,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:20,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:20,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-14T06:16:20,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:20,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565040960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:20,963 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:20,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:20,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:20,964 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T06:16:20,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T06:16:20,965 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T06:16:20,965 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T06:16:20,967 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/616b2d8d3cc1447cadb67c961cc434eb is 50, key is test_row_0/C:col10/1731564980034/Put/seqid=0 2024-11-14T06:16:20,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:20,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565040968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:20,983 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:20,983 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:20,983 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:20,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:20,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565040968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:20,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:20,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:20,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:20,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:20,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565040968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:20,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:20,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:20,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:20,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741878_1054 (size=12151) 2024-11-14T06:16:20,996 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/616b2d8d3cc1447cadb67c961cc434eb 2024-11-14T06:16:21,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/27afff0584af472995f9addfb8b48bc4 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/27afff0584af472995f9addfb8b48bc4 2024-11-14T06:16:21,025 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/27afff0584af472995f9addfb8b48bc4, entries=150, sequenceid=238, filesize=11.9 K 2024-11-14T06:16:21,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/d0dbf4cfdb434649a9e643ee7fc9d8f3 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/d0dbf4cfdb434649a9e643ee7fc9d8f3 2024-11-14T06:16:21,038 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/d0dbf4cfdb434649a9e643ee7fc9d8f3, entries=150, sequenceid=238, filesize=11.9 K 2024-11-14T06:16:21,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/616b2d8d3cc1447cadb67c961cc434eb as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/616b2d8d3cc1447cadb67c961cc434eb 2024-11-14T06:16:21,056 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/616b2d8d3cc1447cadb67c961cc434eb, entries=150, sequenceid=238, filesize=11.9 K 2024-11-14T06:16:21,058 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for d7851abd86d644f97c54179c322b1c1e in 294ms, sequenceid=238, compaction requested=true 2024-11-14T06:16:21,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d7851abd86d644f97c54179c322b1c1e: 2024-11-14T06:16:21,058 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:16:21,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7851abd86d644f97c54179c322b1c1e:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T06:16:21,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:16:21,058 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:16:21,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7851abd86d644f97c54179c322b1c1e:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T06:16:21,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:16:21,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7851abd86d644f97c54179c322b1c1e:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T06:16:21,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:16:21,060 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:16:21,060 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): d7851abd86d644f97c54179c322b1c1e/A is initiating minor compaction (all files) 2024-11-14T06:16:21,060 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7851abd86d644f97c54179c322b1c1e/A in TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:21,060 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/bbf5047c48354d9d99f9cf697cd7335f, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/dbead1a3634346b29b1f95558d23fa76, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/27afff0584af472995f9addfb8b48bc4] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp, totalSize=43.0 K 2024-11-14T06:16:21,060 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:16:21,060 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): d7851abd86d644f97c54179c322b1c1e/B is initiating minor compaction (all files) 2024-11-14T06:16:21,061 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7851abd86d644f97c54179c322b1c1e/B in TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:21,061 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/87d78efde7e949ea8a5a3dc793592d3c, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/ebe2c26aa8e64e5daf006f8bbacc1fb2, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/d0dbf4cfdb434649a9e643ee7fc9d8f3] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp, totalSize=36.0 K 2024-11-14T06:16:21,061 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbf5047c48354d9d99f9cf697cd7335f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731564979577 2024-11-14T06:16:21,062 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting dbead1a3634346b29b1f95558d23fa76, keycount=300, bloomtype=ROW, size=18.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731564979952 2024-11-14T06:16:21,062 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 87d78efde7e949ea8a5a3dc793592d3c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731564979577 2024-11-14T06:16:21,062 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting ebe2c26aa8e64e5daf006f8bbacc1fb2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731564979952 2024-11-14T06:16:21,062 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27afff0584af472995f9addfb8b48bc4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731564980034 2024-11-14T06:16:21,063 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting d0dbf4cfdb434649a9e643ee7fc9d8f3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731564980034 2024-11-14T06:16:21,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T06:16:21,119 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7851abd86d644f97c54179c322b1c1e#B#compaction#42 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:16:21,119 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/153e629ab80d462a862a6647639dc963 is 50, key is test_row_0/B:col10/1731564980034/Put/seqid=0 2024-11-14T06:16:21,123 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7851abd86d644f97c54179c322b1c1e#A#compaction#43 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:16:21,124 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/915b87c2f5b74ddfabdc6b7218354440 is 50, key is test_row_0/A:col10/1731564980034/Put/seqid=0 2024-11-14T06:16:21,126 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-14T06:16:21,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:21,130 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing d7851abd86d644f97c54179c322b1c1e 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-14T06:16:21,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=A 2024-11-14T06:16:21,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:21,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=B 2024-11-14T06:16:21,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:21,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=C 2024-11-14T06:16:21,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:21,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/e92b56aa17964df59d871ac4b0db286e is 50, key is test_row_0/A:col10/1731564980840/Put/seqid=0 2024-11-14T06:16:21,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741879_1055 (size=12629) 2024-11-14T06:16:21,182 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/153e629ab80d462a862a6647639dc963 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/153e629ab80d462a862a6647639dc963 2024-11-14T06:16:21,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. as already flushing 2024-11-14T06:16:21,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on d7851abd86d644f97c54179c322b1c1e 2024-11-14T06:16:21,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741880_1056 (size=12629) 2024-11-14T06:16:21,208 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d7851abd86d644f97c54179c322b1c1e/B of d7851abd86d644f97c54179c322b1c1e into 153e629ab80d462a862a6647639dc963(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:16:21,208 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7851abd86d644f97c54179c322b1c1e: 2024-11-14T06:16:21,208 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., storeName=d7851abd86d644f97c54179c322b1c1e/B, priority=13, startTime=1731564981058; duration=0sec 2024-11-14T06:16:21,209 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:16:21,209 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7851abd86d644f97c54179c322b1c1e:B 2024-11-14T06:16:21,209 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:16:21,211 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:16:21,211 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): d7851abd86d644f97c54179c322b1c1e/C is initiating minor compaction (all files) 2024-11-14T06:16:21,212 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7851abd86d644f97c54179c322b1c1e/C in TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:21,212 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/bcaca51cb497417fa62e2168fa7ceea4, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/0162a3e74f184f49a26f74983c6ce4b6, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/616b2d8d3cc1447cadb67c961cc434eb] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp, totalSize=36.0 K 2024-11-14T06:16:21,213 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting bcaca51cb497417fa62e2168fa7ceea4, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731564979577 2024-11-14T06:16:21,214 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 0162a3e74f184f49a26f74983c6ce4b6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731564979952 2024-11-14T06:16:21,214 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 616b2d8d3cc1447cadb67c961cc434eb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731564980034 2024-11-14T06:16:21,243 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7851abd86d644f97c54179c322b1c1e#C#compaction#45 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:16:21,243 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/e2edec5524844802a3fc00ed729eb767 is 50, key is test_row_0/C:col10/1731564980034/Put/seqid=0 2024-11-14T06:16:21,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:21,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565041221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:21,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741881_1057 (size=12151) 2024-11-14T06:16:21,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:21,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:21,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:21,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:21,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565041227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:21,247 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:21,247 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:21,247 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:21,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:21,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565041239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:21,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:21,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:21,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:21,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:21,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565041245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:21,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:21,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:21,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:21,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:21,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565041241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:21,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:21,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:21,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:21,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741882_1058 (size=12629) 2024-11-14T06:16:21,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T06:16:21,287 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/e2edec5524844802a3fc00ed729eb767 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/e2edec5524844802a3fc00ed729eb767 2024-11-14T06:16:21,301 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d7851abd86d644f97c54179c322b1c1e/C of d7851abd86d644f97c54179c322b1c1e into e2edec5524844802a3fc00ed729eb767(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:16:21,301 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7851abd86d644f97c54179c322b1c1e: 2024-11-14T06:16:21,301 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., storeName=d7851abd86d644f97c54179c322b1c1e/C, priority=13, startTime=1731564981059; duration=0sec 2024-11-14T06:16:21,301 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:16:21,301 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7851abd86d644f97c54179c322b1c1e:C 2024-11-14T06:16:21,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:21,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565041347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:21,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:21,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:21,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:21,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:21,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565041358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:21,375 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:21,375 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:21,375 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:21,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:21,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:21,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565041367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:21,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565041367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:21,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:21,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:21,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:21,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:21,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:21,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:21,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:21,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565041379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:21,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:21,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:21,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:21,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:21,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565041579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:21,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:21,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:21,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:21,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T06:16:21,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:21,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565041588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:21,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:21,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:21,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:21,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:21,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565041588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:21,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:21,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:21,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:21,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:21,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565041588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:21,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:21,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:21,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:21,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:21,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565041598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:21,606 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:21,606 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:21,606 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:21,616 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/915b87c2f5b74ddfabdc6b7218354440 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/915b87c2f5b74ddfabdc6b7218354440 2024-11-14T06:16:21,627 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d7851abd86d644f97c54179c322b1c1e/A of d7851abd86d644f97c54179c322b1c1e into 915b87c2f5b74ddfabdc6b7218354440(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:16:21,627 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7851abd86d644f97c54179c322b1c1e: 2024-11-14T06:16:21,627 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., storeName=d7851abd86d644f97c54179c322b1c1e/A, priority=13, startTime=1731564981058; duration=0sec 2024-11-14T06:16:21,627 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:16:21,627 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7851abd86d644f97c54179c322b1c1e:A 2024-11-14T06:16:21,647 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/e92b56aa17964df59d871ac4b0db286e 2024-11-14T06:16:21,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/5193d475234e4b579aa92a33fad66a92 is 50, key is test_row_0/B:col10/1731564980840/Put/seqid=0 2024-11-14T06:16:21,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741883_1059 (size=12151) 2024-11-14T06:16:21,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:21,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565041887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:21,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:21,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:21,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:21,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:21,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565041900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:21,903 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:21,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:21,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:21,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:21,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565041898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:21,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:21,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:21,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:21,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:21,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565041907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:21,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:21,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:21,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:21,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:21,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565041919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:21,934 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:21,934 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:21,934 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:22,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T06:16:22,142 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/5193d475234e4b579aa92a33fad66a92 2024-11-14T06:16:22,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/6cb882d79d8f4c5b84ee0f1667c79315 is 50, key is test_row_0/C:col10/1731564980840/Put/seqid=0 2024-11-14T06:16:22,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741884_1060 (size=12151) 2024-11-14T06:16:22,223 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/6cb882d79d8f4c5b84ee0f1667c79315 2024-11-14T06:16:22,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/e92b56aa17964df59d871ac4b0db286e as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/e92b56aa17964df59d871ac4b0db286e 2024-11-14T06:16:22,243 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/e92b56aa17964df59d871ac4b0db286e, entries=150, sequenceid=256, filesize=11.9 K 2024-11-14T06:16:22,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/5193d475234e4b579aa92a33fad66a92 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/5193d475234e4b579aa92a33fad66a92 2024-11-14T06:16:22,260 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/5193d475234e4b579aa92a33fad66a92, entries=150, sequenceid=256, filesize=11.9 K 2024-11-14T06:16:22,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/6cb882d79d8f4c5b84ee0f1667c79315 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/6cb882d79d8f4c5b84ee0f1667c79315 2024-11-14T06:16:22,271 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/6cb882d79d8f4c5b84ee0f1667c79315, entries=150, sequenceid=256, filesize=11.9 K 2024-11-14T06:16:22,273 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for d7851abd86d644f97c54179c322b1c1e in 1145ms, sequenceid=256, compaction requested=false 2024-11-14T06:16:22,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for d7851abd86d644f97c54179c322b1c1e: 2024-11-14T06:16:22,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:22,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-14T06:16:22,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-14T06:16:22,283 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-14T06:16:22,284 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3130 sec 2024-11-14T06:16:22,294 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.3300 sec 2024-11-14T06:16:22,411 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d7851abd86d644f97c54179c322b1c1e 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-14T06:16:22,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=A 2024-11-14T06:16:22,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:22,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=B 2024-11-14T06:16:22,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:22,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=C 2024-11-14T06:16:22,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:22,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on d7851abd86d644f97c54179c322b1c1e 2024-11-14T06:16:22,423 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/ea919ccc497d4edf89308816b93fffb9 is 50, key is test_row_0/A:col10/1731564981222/Put/seqid=0 2024-11-14T06:16:22,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741885_1061 (size=14741) 2024-11-14T06:16:22,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:22,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565042447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:22,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:22,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:22,466 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/ea919ccc497d4edf89308816b93fffb9 2024-11-14T06:16:22,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:22,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:22,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565042449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:22,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:22,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:22,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:22,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:22,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565042462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:22,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:22,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565042462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:22,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:22,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565042465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:22,478 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:22,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:22,478 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:22,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:22,478 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:22,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:22,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:22,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:22,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:22,506 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/ad0695ccf950460cba14cfa80724d393 is 50, key is test_row_0/B:col10/1731564981222/Put/seqid=0 2024-11-14T06:16:22,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741886_1062 (size=12301) 2024-11-14T06:16:22,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:22,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565042581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:22,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:22,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:22,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:22,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:22,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565042578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:22,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:22,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:22,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:22,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:22,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565042589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:22,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:22,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:22,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:22,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:22,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565042589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:22,609 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:22,609 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:22,609 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:22,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:22,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565042589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:22,610 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:22,610 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:22,610 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:22,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:22,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565042797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:22,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:22,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:22,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:22,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:22,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565042810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:22,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:22,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:22,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:22,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:22,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565042818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:22,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:22,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:22,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:22,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:22,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565042819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:22,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:22,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565042818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:22,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:22,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:22,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:22,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:22,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:22,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:22,988 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/ad0695ccf950460cba14cfa80724d393 2024-11-14T06:16:23,010 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/20628b4b0bd64a058f96c6d7e788a90d is 50, key is test_row_0/C:col10/1731564981222/Put/seqid=0 2024-11-14T06:16:23,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741887_1063 (size=12301) 2024-11-14T06:16:23,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T06:16:23,109 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:16:23,110 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T06:16:23,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-14T06:16:23,113 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T06:16:23,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-14T06:16:23,114 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T06:16:23,115 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T06:16:23,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:23,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565043117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:23,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:23,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:23,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:23,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:23,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565043127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:23,129 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:23,129 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:23,129 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:23,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:23,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565043138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:23,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:23,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:23,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:23,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:23,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565043148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:23,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:23,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:23,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:23,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:23,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565043148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:23,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:23,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:23,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:23,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-14T06:16:23,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-14T06:16:23,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:23,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. as already flushing 2024-11-14T06:16:23,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:23,277 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:23,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:23,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:23,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-14T06:16:23,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-14T06:16:23,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:23,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. as already flushing 2024-11-14T06:16:23,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:23,437 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:23,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:23,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:23,471 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/20628b4b0bd64a058f96c6d7e788a90d 2024-11-14T06:16:23,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/ea919ccc497d4edf89308816b93fffb9 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/ea919ccc497d4edf89308816b93fffb9 2024-11-14T06:16:23,499 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/ea919ccc497d4edf89308816b93fffb9, entries=200, sequenceid=274, filesize=14.4 K 2024-11-14T06:16:23,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/ad0695ccf950460cba14cfa80724d393 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/ad0695ccf950460cba14cfa80724d393 2024-11-14T06:16:23,524 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/ad0695ccf950460cba14cfa80724d393, entries=150, sequenceid=274, filesize=12.0 K 2024-11-14T06:16:23,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/20628b4b0bd64a058f96c6d7e788a90d as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/20628b4b0bd64a058f96c6d7e788a90d 2024-11-14T06:16:23,547 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/20628b4b0bd64a058f96c6d7e788a90d, entries=150, sequenceid=274, filesize=12.0 K 2024-11-14T06:16:23,549 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for d7851abd86d644f97c54179c322b1c1e in 1138ms, sequenceid=274, compaction requested=true 2024-11-14T06:16:23,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d7851abd86d644f97c54179c322b1c1e: 2024-11-14T06:16:23,549 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:16:23,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7851abd86d644f97c54179c322b1c1e:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T06:16:23,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:16:23,550 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:16:23,551 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39521 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:16:23,551 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37081 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:16:23,551 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): d7851abd86d644f97c54179c322b1c1e/A is initiating minor compaction (all files) 2024-11-14T06:16:23,551 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): d7851abd86d644f97c54179c322b1c1e/B is initiating minor compaction (all files) 2024-11-14T06:16:23,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7851abd86d644f97c54179c322b1c1e:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T06:16:23,551 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7851abd86d644f97c54179c322b1c1e/B in TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:23,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:16:23,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7851abd86d644f97c54179c322b1c1e:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T06:16:23,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:16:23,551 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/153e629ab80d462a862a6647639dc963, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/5193d475234e4b579aa92a33fad66a92, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/ad0695ccf950460cba14cfa80724d393] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp, totalSize=36.2 K 2024-11-14T06:16:23,552 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7851abd86d644f97c54179c322b1c1e/A in TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:23,552 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/915b87c2f5b74ddfabdc6b7218354440, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/e92b56aa17964df59d871ac4b0db286e, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/ea919ccc497d4edf89308816b93fffb9] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp, totalSize=38.6 K 2024-11-14T06:16:23,552 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 153e629ab80d462a862a6647639dc963, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731564980034 2024-11-14T06:16:23,552 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 915b87c2f5b74ddfabdc6b7218354440, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731564980034 2024-11-14T06:16:23,553 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 5193d475234e4b579aa92a33fad66a92, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731564980817 2024-11-14T06:16:23,553 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting e92b56aa17964df59d871ac4b0db286e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731564980817 2024-11-14T06:16:23,553 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting ad0695ccf950460cba14cfa80724d393, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731564981222 2024-11-14T06:16:23,553 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea919ccc497d4edf89308816b93fffb9, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731564981222 2024-11-14T06:16:23,581 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7851abd86d644f97c54179c322b1c1e#A#compaction#51 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:16:23,581 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/e4dc9babe1184bef8cbe6a7498b437ac is 50, key is test_row_0/A:col10/1731564981222/Put/seqid=0 2024-11-14T06:16:23,591 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7851abd86d644f97c54179c322b1c1e#B#compaction#52 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:16:23,591 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/91f78e57640b42c59469054125e7bbb3 is 50, key is test_row_0/B:col10/1731564981222/Put/seqid=0 2024-11-14T06:16:23,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-14T06:16:23,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:23,604 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing d7851abd86d644f97c54179c322b1c1e 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-14T06:16:23,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=A 2024-11-14T06:16:23,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:23,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=B 2024-11-14T06:16:23,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:23,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=C 2024-11-14T06:16:23,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:23,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on d7851abd86d644f97c54179c322b1c1e 2024-11-14T06:16:23,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. as already flushing 2024-11-14T06:16:23,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741888_1064 (size=12881) 2024-11-14T06:16:23,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/52bc365df5a04c08b1ff224e72c120a1 is 50, key is test_row_0/A:col10/1731564982453/Put/seqid=0 2024-11-14T06:16:23,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741889_1065 (size=12881) 2024-11-14T06:16:23,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:23,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565043685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:23,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:23,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:23,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:23,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:23,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565043687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:23,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:23,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565043690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:23,717 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:23,717 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:23,717 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:23,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:23,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:23,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:23,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741890_1066 (size=12301) 2024-11-14T06:16:23,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:23,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565043712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:23,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:23,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565043714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:23,735 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/52bc365df5a04c08b1ff224e72c120a1 2024-11-14T06:16:23,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:23,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:23,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:23,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:23,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:23,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:23,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-14T06:16:23,770 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/91f78e57640b42c59469054125e7bbb3 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/91f78e57640b42c59469054125e7bbb3 2024-11-14T06:16:23,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/0dde91185dae4a5baccc13edb989cd2a is 50, key is test_row_0/B:col10/1731564982453/Put/seqid=0 2024-11-14T06:16:23,781 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d7851abd86d644f97c54179c322b1c1e/B of d7851abd86d644f97c54179c322b1c1e into 91f78e57640b42c59469054125e7bbb3(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:16:23,781 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7851abd86d644f97c54179c322b1c1e: 2024-11-14T06:16:23,781 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., storeName=d7851abd86d644f97c54179c322b1c1e/B, priority=13, startTime=1731564983550; duration=0sec 2024-11-14T06:16:23,782 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:16:23,782 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7851abd86d644f97c54179c322b1c1e:B 2024-11-14T06:16:23,782 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:16:23,783 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37081 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:16:23,784 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): d7851abd86d644f97c54179c322b1c1e/C is initiating minor compaction (all files) 2024-11-14T06:16:23,784 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7851abd86d644f97c54179c322b1c1e/C in TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:23,784 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/e2edec5524844802a3fc00ed729eb767, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/6cb882d79d8f4c5b84ee0f1667c79315, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/20628b4b0bd64a058f96c6d7e788a90d] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp, totalSize=36.2 K 2024-11-14T06:16:23,785 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting e2edec5524844802a3fc00ed729eb767, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731564980034 2024-11-14T06:16:23,786 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 6cb882d79d8f4c5b84ee0f1667c79315, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731564980817 2024-11-14T06:16:23,786 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 20628b4b0bd64a058f96c6d7e788a90d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731564981222 2024-11-14T06:16:23,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:23,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565043817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:23,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:23,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:23,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:23,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:23,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565043828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:23,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:23,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:23,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:23,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741891_1067 (size=12301) 2024-11-14T06:16:23,841 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7851abd86d644f97c54179c322b1c1e#C#compaction#55 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:16:23,842 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/bb35d28de4fe4c8ab7a94a99bac56bd1 is 50, key is test_row_0/C:col10/1731564981222/Put/seqid=0 2024-11-14T06:16:23,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:23,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565043830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:23,846 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:23,846 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:23,846 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:23,856 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/0dde91185dae4a5baccc13edb989cd2a 2024-11-14T06:16:23,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:23,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565043848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:23,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:23,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:23,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:23,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:23,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565043848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:23,861 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:23,861 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:23,861 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:23,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/86a4590700f64bf98230d4c73d7571c2 is 50, key is test_row_0/C:col10/1731564982453/Put/seqid=0 2024-11-14T06:16:23,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741892_1068 (size=12881) 2024-11-14T06:16:23,938 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/bb35d28de4fe4c8ab7a94a99bac56bd1 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/bb35d28de4fe4c8ab7a94a99bac56bd1 2024-11-14T06:16:23,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741893_1069 (size=12301) 2024-11-14T06:16:23,962 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d7851abd86d644f97c54179c322b1c1e/C of d7851abd86d644f97c54179c322b1c1e into bb35d28de4fe4c8ab7a94a99bac56bd1(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:16:23,963 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7851abd86d644f97c54179c322b1c1e: 2024-11-14T06:16:23,963 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., storeName=d7851abd86d644f97c54179c322b1c1e/C, priority=13, startTime=1731564983551; duration=0sec 2024-11-14T06:16:23,963 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/86a4590700f64bf98230d4c73d7571c2 2024-11-14T06:16:23,970 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:16:23,970 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7851abd86d644f97c54179c322b1c1e:C 2024-11-14T06:16:23,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/52bc365df5a04c08b1ff224e72c120a1 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/52bc365df5a04c08b1ff224e72c120a1 2024-11-14T06:16:23,989 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/52bc365df5a04c08b1ff224e72c120a1, entries=150, sequenceid=291, filesize=12.0 K 2024-11-14T06:16:23,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/0dde91185dae4a5baccc13edb989cd2a as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/0dde91185dae4a5baccc13edb989cd2a 2024-11-14T06:16:24,000 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/0dde91185dae4a5baccc13edb989cd2a, entries=150, sequenceid=291, filesize=12.0 K 2024-11-14T06:16:24,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/86a4590700f64bf98230d4c73d7571c2 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/86a4590700f64bf98230d4c73d7571c2 2024-11-14T06:16:24,023 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/86a4590700f64bf98230d4c73d7571c2, entries=150, sequenceid=291, filesize=12.0 K 2024-11-14T06:16:24,033 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for d7851abd86d644f97c54179c322b1c1e in 429ms, sequenceid=291, compaction requested=false 2024-11-14T06:16:24,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for d7851abd86d644f97c54179c322b1c1e: 2024-11-14T06:16:24,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:24,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-14T06:16:24,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-14T06:16:24,044 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-14T06:16:24,044 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 926 msec 2024-11-14T06:16:24,050 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 935 msec 2024-11-14T06:16:24,053 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d7851abd86d644f97c54179c322b1c1e 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-14T06:16:24,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on d7851abd86d644f97c54179c322b1c1e 2024-11-14T06:16:24,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=A 2024-11-14T06:16:24,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:24,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=B 2024-11-14T06:16:24,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:24,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=C 2024-11-14T06:16:24,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:24,060 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/e4dc9babe1184bef8cbe6a7498b437ac as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/e4dc9babe1184bef8cbe6a7498b437ac 2024-11-14T06:16:24,077 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d7851abd86d644f97c54179c322b1c1e/A of d7851abd86d644f97c54179c322b1c1e into e4dc9babe1184bef8cbe6a7498b437ac(size=12.6 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:16:24,077 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7851abd86d644f97c54179c322b1c1e: 2024-11-14T06:16:24,077 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., storeName=d7851abd86d644f97c54179c322b1c1e/A, priority=13, startTime=1731564983549; duration=0sec 2024-11-14T06:16:24,077 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:16:24,080 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/ac049d36ac8743da968b08e3b7f247da is 50, key is test_row_0/A:col10/1731564984048/Put/seqid=0 2024-11-14T06:16:24,081 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7851abd86d644f97c54179c322b1c1e:A 2024-11-14T06:16:24,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:24,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565044099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:24,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:24,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:24,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:24,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:24,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565044101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:24,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:24,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:24,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:24,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:24,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565044102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:24,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:24,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565044103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:24,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:24,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:24,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:24,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:24,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:24,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:24,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:24,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741894_1070 (size=14741) 2024-11-14T06:16:24,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565044105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:24,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:24,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:24,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:24,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:24,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565044218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:24,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:24,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565044218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:24,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:24,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:24,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:24,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:24,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565044218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:24,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:24,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565044218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:24,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:24,224 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:24,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:24,224 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:24,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:24,224 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:24,225 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:24,225 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:24,225 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:24,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:24,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565044227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:24,233 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:24,233 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:24,233 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:24,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-14T06:16:24,259 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:16:24,260 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T06:16:24,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-14T06:16:24,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-14T06:16:24,263 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T06:16:24,264 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T06:16:24,265 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T06:16:24,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-14T06:16:24,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-14T06:16:24,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:24,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. as already flushing 2024-11-14T06:16:24,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:24,421 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:24,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:24,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:24,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:24,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565044427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:24,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:24,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:24,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:24,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:24,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565044428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:24,430 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:24,430 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:24,430 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:24,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:24,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565044437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:24,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:24,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:24,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:24,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:24,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565044437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:24,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:24,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565044437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:24,445 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:24,445 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:24,445 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:24,449 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:24,449 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:24,449 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:24,518 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/ac049d36ac8743da968b08e3b7f247da 2024-11-14T06:16:24,527 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/b207d6b46cb14fe2a1ee9a61b4f8f714 is 50, key is test_row_0/B:col10/1731564984048/Put/seqid=0 2024-11-14T06:16:24,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741895_1071 (size=12301) 2024-11-14T06:16:24,532 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/b207d6b46cb14fe2a1ee9a61b4f8f714 2024-11-14T06:16:24,568 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/9fc1f894f35e4167b36dccdcb330aef0 is 50, key is test_row_0/C:col10/1731564984048/Put/seqid=0 2024-11-14T06:16:24,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-14T06:16:24,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:24,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. as already flushing 2024-11-14T06:16:24,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:24,576 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:24,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:24,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-14T06:16:24,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:24,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741896_1072 (size=12301) 2024-11-14T06:16:24,597 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/9fc1f894f35e4167b36dccdcb330aef0 2024-11-14T06:16:24,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/ac049d36ac8743da968b08e3b7f247da as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/ac049d36ac8743da968b08e3b7f247da 2024-11-14T06:16:24,627 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/ac049d36ac8743da968b08e3b7f247da, entries=200, sequenceid=310, filesize=14.4 K 2024-11-14T06:16:24,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/b207d6b46cb14fe2a1ee9a61b4f8f714 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/b207d6b46cb14fe2a1ee9a61b4f8f714 2024-11-14T06:16:24,637 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/b207d6b46cb14fe2a1ee9a61b4f8f714, entries=150, sequenceid=310, filesize=12.0 K 2024-11-14T06:16:24,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/9fc1f894f35e4167b36dccdcb330aef0 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/9fc1f894f35e4167b36dccdcb330aef0 2024-11-14T06:16:24,655 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/9fc1f894f35e4167b36dccdcb330aef0, entries=150, sequenceid=310, filesize=12.0 K 2024-11-14T06:16:24,663 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for d7851abd86d644f97c54179c322b1c1e in 610ms, sequenceid=310, compaction requested=true 2024-11-14T06:16:24,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d7851abd86d644f97c54179c322b1c1e: 2024-11-14T06:16:24,663 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:16:24,665 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39923 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:16:24,665 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): d7851abd86d644f97c54179c322b1c1e/A is initiating minor compaction (all files) 2024-11-14T06:16:24,665 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7851abd86d644f97c54179c322b1c1e/A in TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:24,665 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/e4dc9babe1184bef8cbe6a7498b437ac, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/52bc365df5a04c08b1ff224e72c120a1, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/ac049d36ac8743da968b08e3b7f247da] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp, totalSize=39.0 K 2024-11-14T06:16:24,666 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4dc9babe1184bef8cbe6a7498b437ac, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731564981222 2024-11-14T06:16:24,667 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52bc365df5a04c08b1ff224e72c120a1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731564982444 2024-11-14T06:16:24,667 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac049d36ac8743da968b08e3b7f247da, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731564983663 2024-11-14T06:16:24,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7851abd86d644f97c54179c322b1c1e:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T06:16:24,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:16:24,670 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:16:24,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7851abd86d644f97c54179c322b1c1e:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T06:16:24,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:16:24,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7851abd86d644f97c54179c322b1c1e:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T06:16:24,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:16:24,672 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:16:24,672 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): d7851abd86d644f97c54179c322b1c1e/B is initiating minor compaction (all files) 2024-11-14T06:16:24,672 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7851abd86d644f97c54179c322b1c1e/B in TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:24,672 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/91f78e57640b42c59469054125e7bbb3, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/0dde91185dae4a5baccc13edb989cd2a, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/b207d6b46cb14fe2a1ee9a61b4f8f714] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp, totalSize=36.6 K 2024-11-14T06:16:24,673 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 91f78e57640b42c59469054125e7bbb3, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731564981222 2024-11-14T06:16:24,674 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 0dde91185dae4a5baccc13edb989cd2a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731564982444 2024-11-14T06:16:24,674 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting b207d6b46cb14fe2a1ee9a61b4f8f714, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731564983699 2024-11-14T06:16:24,687 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7851abd86d644f97c54179c322b1c1e#A#compaction#60 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:16:24,689 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/c9623358f1e845d187355d2f77d0816e is 50, key is test_row_0/A:col10/1731564984048/Put/seqid=0 2024-11-14T06:16:24,698 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7851abd86d644f97c54179c322b1c1e#B#compaction#61 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:16:24,699 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/ac0d1c658cae4284a1ada552615673eb is 50, key is test_row_0/B:col10/1731564984048/Put/seqid=0 2024-11-14T06:16:24,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741897_1073 (size=12983) 2024-11-14T06:16:24,744 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-14T06:16:24,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:24,745 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing d7851abd86d644f97c54179c322b1c1e 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-14T06:16:24,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on d7851abd86d644f97c54179c322b1c1e 2024-11-14T06:16:24,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. as already flushing 2024-11-14T06:16:24,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=A 2024-11-14T06:16:24,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:24,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741898_1074 (size=12983) 2024-11-14T06:16:24,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=B 2024-11-14T06:16:24,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:24,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=C 2024-11-14T06:16:24,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:24,759 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/c9623358f1e845d187355d2f77d0816e as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/c9623358f1e845d187355d2f77d0816e 2024-11-14T06:16:24,768 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/ac0d1c658cae4284a1ada552615673eb as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/ac0d1c658cae4284a1ada552615673eb 2024-11-14T06:16:24,777 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d7851abd86d644f97c54179c322b1c1e/A of d7851abd86d644f97c54179c322b1c1e into c9623358f1e845d187355d2f77d0816e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:16:24,777 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7851abd86d644f97c54179c322b1c1e: 2024-11-14T06:16:24,777 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., storeName=d7851abd86d644f97c54179c322b1c1e/A, priority=13, startTime=1731564984663; duration=0sec 2024-11-14T06:16:24,777 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:16:24,777 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7851abd86d644f97c54179c322b1c1e:A 2024-11-14T06:16:24,777 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:16:24,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/e538752ca4d04a529710b8f50772d642 is 50, key is test_row_0/A:col10/1731564984100/Put/seqid=0 2024-11-14T06:16:24,781 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d7851abd86d644f97c54179c322b1c1e/B of d7851abd86d644f97c54179c322b1c1e into ac0d1c658cae4284a1ada552615673eb(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:16:24,781 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7851abd86d644f97c54179c322b1c1e: 2024-11-14T06:16:24,782 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., storeName=d7851abd86d644f97c54179c322b1c1e/B, priority=13, startTime=1731564984669; duration=0sec 2024-11-14T06:16:24,782 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:16:24,782 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7851abd86d644f97c54179c322b1c1e:B 2024-11-14T06:16:24,788 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:16:24,788 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): d7851abd86d644f97c54179c322b1c1e/C is initiating minor compaction (all files) 2024-11-14T06:16:24,788 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7851abd86d644f97c54179c322b1c1e/C in TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:24,788 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/bb35d28de4fe4c8ab7a94a99bac56bd1, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/86a4590700f64bf98230d4c73d7571c2, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/9fc1f894f35e4167b36dccdcb330aef0] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp, totalSize=36.6 K 2024-11-14T06:16:24,789 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb35d28de4fe4c8ab7a94a99bac56bd1, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731564981222 2024-11-14T06:16:24,791 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86a4590700f64bf98230d4c73d7571c2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731564982444 2024-11-14T06:16:24,791 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9fc1f894f35e4167b36dccdcb330aef0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731564983699 2024-11-14T06:16:24,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:24,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565044813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:24,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:24,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565044814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:24,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:24,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:24,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:24,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:24,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:24,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:24,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:24,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565044817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:24,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:24,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:24,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:24,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:24,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565044818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:24,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:24,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:24,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:24,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:24,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565044825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:24,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:24,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:24,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:24,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741899_1075 (size=17181) 2024-11-14T06:16:24,840 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/e538752ca4d04a529710b8f50772d642 2024-11-14T06:16:24,851 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7851abd86d644f97c54179c322b1c1e#C#compaction#63 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:16:24,851 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/c6287bb5179b466f9e8a78fc689fce5d is 50, key is test_row_0/C:col10/1731564984048/Put/seqid=0 2024-11-14T06:16:24,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/eecf806f76dd4cab8d0f1300fe054e48 is 50, key is test_row_0/B:col10/1731564984100/Put/seqid=0 2024-11-14T06:16:24,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-14T06:16:24,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741900_1076 (size=12983) 2024-11-14T06:16:24,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:24,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565044938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:24,940 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:24,940 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:24,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:24,940 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:24,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565044939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:24,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:24,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:24,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565044939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:24,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:24,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:24,942 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:24,942 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:24,943 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:24,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741901_1077 (size=12301) 2024-11-14T06:16:24,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:24,947 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/eecf806f76dd4cab8d0f1300fe054e48 2024-11-14T06:16:24,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565044944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:24,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:24,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:24,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:24,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:24,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565044944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:24,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:24,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:24,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:24,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/85306f34cdd44ac49b949333a326e121 is 50, key is test_row_0/C:col10/1731564984100/Put/seqid=0 2024-11-14T06:16:24,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741902_1078 (size=12301) 2024-11-14T06:16:24,996 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/85306f34cdd44ac49b949333a326e121 2024-11-14T06:16:25,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/e538752ca4d04a529710b8f50772d642 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/e538752ca4d04a529710b8f50772d642 2024-11-14T06:16:25,019 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/e538752ca4d04a529710b8f50772d642, entries=250, sequenceid=328, filesize=16.8 K 2024-11-14T06:16:25,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/eecf806f76dd4cab8d0f1300fe054e48 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/eecf806f76dd4cab8d0f1300fe054e48 2024-11-14T06:16:25,031 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/eecf806f76dd4cab8d0f1300fe054e48, entries=150, sequenceid=328, filesize=12.0 K 2024-11-14T06:16:25,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/85306f34cdd44ac49b949333a326e121 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/85306f34cdd44ac49b949333a326e121 2024-11-14T06:16:25,041 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/85306f34cdd44ac49b949333a326e121, entries=150, sequenceid=328, filesize=12.0 K 2024-11-14T06:16:25,043 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for d7851abd86d644f97c54179c322b1c1e in 297ms, sequenceid=328, compaction requested=false 2024-11-14T06:16:25,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for d7851abd86d644f97c54179c322b1c1e: 2024-11-14T06:16:25,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:25,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-14T06:16:25,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-14T06:16:25,050 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-14T06:16:25,050 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 781 msec 2024-11-14T06:16:25,054 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 792 msec 2024-11-14T06:16:25,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on d7851abd86d644f97c54179c322b1c1e 2024-11-14T06:16:25,155 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d7851abd86d644f97c54179c322b1c1e 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-14T06:16:25,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=A 2024-11-14T06:16:25,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:25,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=B 2024-11-14T06:16:25,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:25,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=C 2024-11-14T06:16:25,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:25,176 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/4b8f99c377544b619470832f84be5991 is 50, key is test_row_0/A:col10/1731564985151/Put/seqid=0 2024-11-14T06:16:25,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:25,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565045181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:25,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741903_1079 (size=12301) 2024-11-14T06:16:25,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:25,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:25,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565045181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:25,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565045186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:25,207 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/4b8f99c377544b619470832f84be5991 2024-11-14T06:16:25,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:25,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565045199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:25,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:25,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:25,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:25,213 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:25,213 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:25,213 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:25,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:25,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565045205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:25,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:25,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:25,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:25,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:25,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:25,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:25,217 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:25,217 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:25,217 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:25,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/cbd1d52572624906b27eac2136e31b1c is 50, key is test_row_0/B:col10/1731564985151/Put/seqid=0 2024-11-14T06:16:25,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741904_1080 (size=12301) 2024-11-14T06:16:25,275 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/cbd1d52572624906b27eac2136e31b1c 2024-11-14T06:16:25,310 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/9bbed59575a7411cb7c02d14b1c453ac is 50, key is test_row_0/C:col10/1731564985151/Put/seqid=0 2024-11-14T06:16:25,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:25,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565045317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:25,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:25,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:25,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:25,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:25,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565045319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:25,326 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:25,326 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:25,326 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:25,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741905_1081 (size=12301) 2024-11-14T06:16:25,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:25,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565045329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:25,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:25,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:25,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:25,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:25,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565045329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:25,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:25,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:25,340 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/c6287bb5179b466f9e8a78fc689fce5d as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/c6287bb5179b466f9e8a78fc689fce5d 2024-11-14T06:16:25,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:25,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:25,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565045329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:25,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:25,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:25,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:25,352 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d7851abd86d644f97c54179c322b1c1e/C of d7851abd86d644f97c54179c322b1c1e into c6287bb5179b466f9e8a78fc689fce5d(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:16:25,352 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7851abd86d644f97c54179c322b1c1e: 2024-11-14T06:16:25,352 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., storeName=d7851abd86d644f97c54179c322b1c1e/C, priority=13, startTime=1731564984670; duration=0sec 2024-11-14T06:16:25,352 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:16:25,353 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7851abd86d644f97c54179c322b1c1e:C 2024-11-14T06:16:25,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-14T06:16:25,398 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:16:25,398 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T06:16:25,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-14T06:16:25,401 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T06:16:25,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-14T06:16:25,405 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T06:16:25,405 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T06:16:25,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-14T06:16:25,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:25,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565045528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:25,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:25,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:25,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:25,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:25,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565045537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:25,538 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:25,538 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:25,538 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:25,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:25,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:25,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565045548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:25,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565045547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:25,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:25,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565045548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:25,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:25,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:25,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:25,555 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:25,555 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:25,555 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:25,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:25,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:25,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:25,558 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-14T06:16:25,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:25,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. as already flushing 2024-11-14T06:16:25,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:25,558 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:25,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:25,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:25,714 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-14T06:16:25,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:25,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. as already flushing 2024-11-14T06:16:25,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:25,715 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:25,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:25,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:25,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-14T06:16:25,734 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/9bbed59575a7411cb7c02d14b1c453ac 2024-11-14T06:16:25,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/4b8f99c377544b619470832f84be5991 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/4b8f99c377544b619470832f84be5991 2024-11-14T06:16:25,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/4b8f99c377544b619470832f84be5991, entries=150, sequenceid=347, filesize=12.0 K 2024-11-14T06:16:25,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/cbd1d52572624906b27eac2136e31b1c as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/cbd1d52572624906b27eac2136e31b1c 2024-11-14T06:16:25,764 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/cbd1d52572624906b27eac2136e31b1c, entries=150, sequenceid=347, filesize=12.0 K 2024-11-14T06:16:25,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/9bbed59575a7411cb7c02d14b1c453ac as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/9bbed59575a7411cb7c02d14b1c453ac 2024-11-14T06:16:25,775 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/9bbed59575a7411cb7c02d14b1c453ac, entries=150, sequenceid=347, filesize=12.0 K 2024-11-14T06:16:25,776 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for d7851abd86d644f97c54179c322b1c1e in 622ms, sequenceid=347, compaction requested=true 2024-11-14T06:16:25,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d7851abd86d644f97c54179c322b1c1e: 2024-11-14T06:16:25,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7851abd86d644f97c54179c322b1c1e:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T06:16:25,777 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:16:25,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:16:25,777 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:16:25,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7851abd86d644f97c54179c322b1c1e:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T06:16:25,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:16:25,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7851abd86d644f97c54179c322b1c1e:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T06:16:25,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:16:25,779 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:16:25,779 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): d7851abd86d644f97c54179c322b1c1e/B is initiating minor compaction (all files) 2024-11-14T06:16:25,779 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7851abd86d644f97c54179c322b1c1e/B in TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:25,779 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42465 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:16:25,779 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): d7851abd86d644f97c54179c322b1c1e/A is initiating minor compaction (all files) 2024-11-14T06:16:25,779 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/ac0d1c658cae4284a1ada552615673eb, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/eecf806f76dd4cab8d0f1300fe054e48, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/cbd1d52572624906b27eac2136e31b1c] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp, totalSize=36.7 K 2024-11-14T06:16:25,779 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7851abd86d644f97c54179c322b1c1e/A in TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:25,779 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/c9623358f1e845d187355d2f77d0816e, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/e538752ca4d04a529710b8f50772d642, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/4b8f99c377544b619470832f84be5991] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp, totalSize=41.5 K 2024-11-14T06:16:25,780 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting ac0d1c658cae4284a1ada552615673eb, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731564983699 2024-11-14T06:16:25,780 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9623358f1e845d187355d2f77d0816e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731564983699 2024-11-14T06:16:25,780 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting eecf806f76dd4cab8d0f1300fe054e48, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731564984100 2024-11-14T06:16:25,780 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting e538752ca4d04a529710b8f50772d642, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731564984097 2024-11-14T06:16:25,781 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b8f99c377544b619470832f84be5991, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731564984823 2024-11-14T06:16:25,781 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting cbd1d52572624906b27eac2136e31b1c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731564984823 2024-11-14T06:16:25,819 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7851abd86d644f97c54179c322b1c1e#A#compaction#69 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:16:25,819 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/eb6b265958ff48f18fb27cf7e284ca02 is 50, key is test_row_0/A:col10/1731564985151/Put/seqid=0 2024-11-14T06:16:25,829 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7851abd86d644f97c54179c322b1c1e#B#compaction#70 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:16:25,829 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/a9babefd97254158a25b1764ee9741fb is 50, key is test_row_0/B:col10/1731564985151/Put/seqid=0 2024-11-14T06:16:25,843 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d7851abd86d644f97c54179c322b1c1e 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-14T06:16:25,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=A 2024-11-14T06:16:25,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:25,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=B 2024-11-14T06:16:25,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:25,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=C 2024-11-14T06:16:25,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:25,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on d7851abd86d644f97c54179c322b1c1e 2024-11-14T06:16:25,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741906_1082 (size=13085) 2024-11-14T06:16:25,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741907_1083 (size=13085) 2024-11-14T06:16:25,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/3c720d0ece964299bf31b9016dd94247 is 50, key is test_row_0/A:col10/1731564985187/Put/seqid=0 2024-11-14T06:16:25,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-14T06:16:25,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:25,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. as already flushing 2024-11-14T06:16:25,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:25,874 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/eb6b265958ff48f18fb27cf7e284ca02 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/eb6b265958ff48f18fb27cf7e284ca02 2024-11-14T06:16:25,874 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:25,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:25,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:25,879 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/a9babefd97254158a25b1764ee9741fb as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/a9babefd97254158a25b1764ee9741fb 2024-11-14T06:16:25,886 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d7851abd86d644f97c54179c322b1c1e/A of d7851abd86d644f97c54179c322b1c1e into eb6b265958ff48f18fb27cf7e284ca02(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:16:25,886 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7851abd86d644f97c54179c322b1c1e: 2024-11-14T06:16:25,886 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., storeName=d7851abd86d644f97c54179c322b1c1e/A, priority=13, startTime=1731564985776; duration=0sec 2024-11-14T06:16:25,886 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:16:25,886 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7851abd86d644f97c54179c322b1c1e:A 2024-11-14T06:16:25,887 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:16:25,894 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:16:25,894 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): d7851abd86d644f97c54179c322b1c1e/C is initiating minor compaction (all files) 2024-11-14T06:16:25,894 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7851abd86d644f97c54179c322b1c1e/C in TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:25,894 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/c6287bb5179b466f9e8a78fc689fce5d, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/85306f34cdd44ac49b949333a326e121, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/9bbed59575a7411cb7c02d14b1c453ac] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp, totalSize=36.7 K 2024-11-14T06:16:25,895 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d7851abd86d644f97c54179c322b1c1e/B of d7851abd86d644f97c54179c322b1c1e into a9babefd97254158a25b1764ee9741fb(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:16:25,895 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7851abd86d644f97c54179c322b1c1e: 2024-11-14T06:16:25,895 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., storeName=d7851abd86d644f97c54179c322b1c1e/B, priority=13, startTime=1731564985777; duration=0sec 2024-11-14T06:16:25,895 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:16:25,895 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7851abd86d644f97c54179c322b1c1e:B 2024-11-14T06:16:25,896 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6287bb5179b466f9e8a78fc689fce5d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731564983699 2024-11-14T06:16:25,896 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85306f34cdd44ac49b949333a326e121, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731564984100 2024-11-14T06:16:25,897 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9bbed59575a7411cb7c02d14b1c453ac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731564984823 2024-11-14T06:16:25,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741908_1084 (size=14741) 2024-11-14T06:16:25,914 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7851abd86d644f97c54179c322b1c1e#C#compaction#72 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:16:25,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:25,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565045904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:25,915 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/3074e07bef3f4ba39489fae3dd75c843 is 50, key is test_row_0/C:col10/1731564985151/Put/seqid=0 2024-11-14T06:16:25,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:25,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:25,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:25,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:25,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565045908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:25,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:25,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:25,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:25,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:25,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565045911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:25,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:25,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:25,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:25,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:25,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565045914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:25,923 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:25,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:25,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:25,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:25,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565045917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:25,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:25,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:25,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:25,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741909_1085 (size=13085) 2024-11-14T06:16:26,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:26,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565046017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:26,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:26,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:26,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:26,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-14T06:16:26,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:26,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565046029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:26,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:26,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:26,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:26,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:26,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565046029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:26,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:26,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:26,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:26,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:26,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565046029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:26,035 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:26,035 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:26,035 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:26,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-14T06:16:26,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:26,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. as already flushing 2024-11-14T06:16:26,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:26,037 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:26,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:26,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:26,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:26,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565046037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:26,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:26,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:26,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:26,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-14T06:16:26,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:26,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. as already flushing 2024-11-14T06:16:26,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:26,195 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:26,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:26,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:26,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:26,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565046228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:26,235 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:26,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:26,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:26,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:26,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565046238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:26,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:26,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:26,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565046239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:26,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565046238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:26,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:26,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:26,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:26,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:26,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:26,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:26,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:26,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:26,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:26,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:26,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565046259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:26,262 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:26,262 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:26,262 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:26,305 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/3c720d0ece964299bf31b9016dd94247 2024-11-14T06:16:26,333 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/aad6656df5da42168acb9eef03242cf8 is 50, key is test_row_0/B:col10/1731564985187/Put/seqid=0 2024-11-14T06:16:26,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-14T06:16:26,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:26,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. as already flushing 2024-11-14T06:16:26,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:26,351 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:26,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:26,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:26,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741910_1086 (size=12301) 2024-11-14T06:16:26,365 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/aad6656df5da42168acb9eef03242cf8 2024-11-14T06:16:26,369 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/3074e07bef3f4ba39489fae3dd75c843 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/3074e07bef3f4ba39489fae3dd75c843 2024-11-14T06:16:26,382 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/bc9e694409584947aaea92a3c1e01e8d is 50, key is test_row_0/C:col10/1731564985187/Put/seqid=0 2024-11-14T06:16:26,385 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d7851abd86d644f97c54179c322b1c1e/C of d7851abd86d644f97c54179c322b1c1e into 3074e07bef3f4ba39489fae3dd75c843(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:16:26,385 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7851abd86d644f97c54179c322b1c1e: 2024-11-14T06:16:26,386 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., storeName=d7851abd86d644f97c54179c322b1c1e/C, priority=13, startTime=1731564985777; duration=0sec 2024-11-14T06:16:26,386 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:16:26,386 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7851abd86d644f97c54179c322b1c1e:C 2024-11-14T06:16:26,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741911_1087 (size=12301) 2024-11-14T06:16:26,409 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/bc9e694409584947aaea92a3c1e01e8d 2024-11-14T06:16:26,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/3c720d0ece964299bf31b9016dd94247 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/3c720d0ece964299bf31b9016dd94247 2024-11-14T06:16:26,426 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/3c720d0ece964299bf31b9016dd94247, entries=200, sequenceid=364, filesize=14.4 K 2024-11-14T06:16:26,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/aad6656df5da42168acb9eef03242cf8 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/aad6656df5da42168acb9eef03242cf8 2024-11-14T06:16:26,435 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/aad6656df5da42168acb9eef03242cf8, entries=150, sequenceid=364, filesize=12.0 K 2024-11-14T06:16:26,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/bc9e694409584947aaea92a3c1e01e8d as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/bc9e694409584947aaea92a3c1e01e8d 2024-11-14T06:16:26,443 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/bc9e694409584947aaea92a3c1e01e8d, entries=150, sequenceid=364, filesize=12.0 K 2024-11-14T06:16:26,445 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for d7851abd86d644f97c54179c322b1c1e in 602ms, sequenceid=364, compaction requested=false 2024-11-14T06:16:26,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d7851abd86d644f97c54179c322b1c1e: 2024-11-14T06:16:26,508 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-14T06:16:26,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:26,511 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing d7851abd86d644f97c54179c322b1c1e 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-14T06:16:26,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=A 2024-11-14T06:16:26,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:26,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=B 2024-11-14T06:16:26,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:26,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=C 2024-11-14T06:16:26,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:26,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/2165039abefd4e1f993da62a4f89bb44 is 50, key is test_row_0/A:col10/1731564985904/Put/seqid=0 2024-11-14T06:16:26,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741912_1088 (size=12301) 2024-11-14T06:16:26,533 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/2165039abefd4e1f993da62a4f89bb44 2024-11-14T06:16:26,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-14T06:16:26,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/9e0435bdfcd043e89bea854a2d7a4110 is 50, key is test_row_0/B:col10/1731564985904/Put/seqid=0 2024-11-14T06:16:26,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on d7851abd86d644f97c54179c322b1c1e 2024-11-14T06:16:26,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. as already flushing 2024-11-14T06:16:26,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741913_1089 (size=12301) 2024-11-14T06:16:26,567 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/9e0435bdfcd043e89bea854a2d7a4110 2024-11-14T06:16:26,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/c3fa25b1cea94818bfe48c384f3051fc is 50, key is test_row_0/C:col10/1731564985904/Put/seqid=0 2024-11-14T06:16:26,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:26,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565046582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:26,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:26,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:26,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:26,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565046584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:26,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:26,594 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:26,594 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:26,594 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:26,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:26,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565046586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:26,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:26,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:26,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:26,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:26,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565046588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:26,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:26,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:26,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:26,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565046588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:26,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:26,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:26,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:26,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:26,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741914_1090 (size=12301) 2024-11-14T06:16:26,638 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/c3fa25b1cea94818bfe48c384f3051fc 2024-11-14T06:16:26,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/2165039abefd4e1f993da62a4f89bb44 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/2165039abefd4e1f993da62a4f89bb44 2024-11-14T06:16:26,657 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/2165039abefd4e1f993da62a4f89bb44, entries=150, sequenceid=384, filesize=12.0 K 2024-11-14T06:16:26,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/9e0435bdfcd043e89bea854a2d7a4110 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/9e0435bdfcd043e89bea854a2d7a4110 2024-11-14T06:16:26,681 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/9e0435bdfcd043e89bea854a2d7a4110, entries=150, sequenceid=384, filesize=12.0 K 2024-11-14T06:16:26,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/c3fa25b1cea94818bfe48c384f3051fc as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/c3fa25b1cea94818bfe48c384f3051fc 2024-11-14T06:16:26,693 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/c3fa25b1cea94818bfe48c384f3051fc, entries=150, sequenceid=384, filesize=12.0 K 2024-11-14T06:16:26,694 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for d7851abd86d644f97c54179c322b1c1e in 186ms, sequenceid=384, compaction requested=true 2024-11-14T06:16:26,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for d7851abd86d644f97c54179c322b1c1e: 2024-11-14T06:16:26,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:26,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-14T06:16:26,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-14T06:16:26,702 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-14T06:16:26,702 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2940 sec 2024-11-14T06:16:26,705 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.3040 sec 2024-11-14T06:16:26,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on d7851abd86d644f97c54179c322b1c1e 2024-11-14T06:16:26,713 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d7851abd86d644f97c54179c322b1c1e 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-14T06:16:26,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=A 2024-11-14T06:16:26,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:26,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=B 2024-11-14T06:16:26,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:26,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=C 2024-11-14T06:16:26,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:26,730 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/742db572e4a3425bb1cd2d97ab808016 is 50, key is test_row_0/A:col10/1731564986699/Put/seqid=0 2024-11-14T06:16:26,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741915_1091 (size=12301) 2024-11-14T06:16:26,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:26,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565046736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:26,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:26,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:26,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:26,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:26,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565046738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:26,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:26,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:26,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:26,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:26,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565046739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:26,751 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:26,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:26,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:26,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:26,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565046747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:26,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:26,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:26,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:26,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:26,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565046748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:26,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:26,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:26,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:26,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:26,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565046858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:26,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:26,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:26,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:26,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:26,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565046858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:26,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:26,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:26,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:26,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:26,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565046858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:26,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:26,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:26,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:26,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:26,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565046859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:26,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:26,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:26,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:26,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:26,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565046868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:26,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:26,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:26,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:27,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:27,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565047068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:27,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:27,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:27,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:27,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:27,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565047068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:27,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:27,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:27,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:27,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:27,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565047068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:27,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:27,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:27,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:27,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:27,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565047068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:27,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:27,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:27,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:27,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:27,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565047077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:27,084 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:27,084 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:27,084 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:27,143 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/742db572e4a3425bb1cd2d97ab808016 2024-11-14T06:16:27,166 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/3242feb0dd8943e5bdc5e89efbf3bc6c is 50, key is test_row_0/B:col10/1731564986699/Put/seqid=0 2024-11-14T06:16:27,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741916_1092 (size=12301) 2024-11-14T06:16:27,214 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/3242feb0dd8943e5bdc5e89efbf3bc6c 2024-11-14T06:16:27,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/4cf065cc93284835a08c11ff91ab262f is 50, key is test_row_0/C:col10/1731564986699/Put/seqid=0 2024-11-14T06:16:27,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741917_1093 (size=12301) 2024-11-14T06:16:27,267 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/4cf065cc93284835a08c11ff91ab262f 2024-11-14T06:16:27,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/742db572e4a3425bb1cd2d97ab808016 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/742db572e4a3425bb1cd2d97ab808016 2024-11-14T06:16:27,295 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/742db572e4a3425bb1cd2d97ab808016, entries=150, sequenceid=402, filesize=12.0 K 2024-11-14T06:16:27,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/3242feb0dd8943e5bdc5e89efbf3bc6c as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/3242feb0dd8943e5bdc5e89efbf3bc6c 2024-11-14T06:16:27,306 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/3242feb0dd8943e5bdc5e89efbf3bc6c, entries=150, sequenceid=402, filesize=12.0 K 2024-11-14T06:16:27,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/4cf065cc93284835a08c11ff91ab262f as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/4cf065cc93284835a08c11ff91ab262f 2024-11-14T06:16:27,316 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/4cf065cc93284835a08c11ff91ab262f, entries=150, sequenceid=402, filesize=12.0 K 2024-11-14T06:16:27,317 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for d7851abd86d644f97c54179c322b1c1e in 604ms, sequenceid=402, compaction requested=true 2024-11-14T06:16:27,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d7851abd86d644f97c54179c322b1c1e: 2024-11-14T06:16:27,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7851abd86d644f97c54179c322b1c1e:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T06:16:27,318 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T06:16:27,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:16:27,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7851abd86d644f97c54179c322b1c1e:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T06:16:27,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:16:27,318 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T06:16:27,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7851abd86d644f97c54179c322b1c1e:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T06:16:27,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:16:27,320 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52428 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T06:16:27,320 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): d7851abd86d644f97c54179c322b1c1e/A is initiating minor compaction (all files) 2024-11-14T06:16:27,320 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7851abd86d644f97c54179c322b1c1e/A in TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:27,320 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/eb6b265958ff48f18fb27cf7e284ca02, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/3c720d0ece964299bf31b9016dd94247, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/2165039abefd4e1f993da62a4f89bb44, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/742db572e4a3425bb1cd2d97ab808016] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp, totalSize=51.2 K 2024-11-14T06:16:27,323 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T06:16:27,323 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): d7851abd86d644f97c54179c322b1c1e/B is initiating minor compaction (all files) 2024-11-14T06:16:27,323 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7851abd86d644f97c54179c322b1c1e/B in TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:27,323 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/a9babefd97254158a25b1764ee9741fb, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/aad6656df5da42168acb9eef03242cf8, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/9e0435bdfcd043e89bea854a2d7a4110, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/3242feb0dd8943e5bdc5e89efbf3bc6c] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp, totalSize=48.8 K 2024-11-14T06:16:27,324 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb6b265958ff48f18fb27cf7e284ca02, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731564984823 2024-11-14T06:16:27,325 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting a9babefd97254158a25b1764ee9741fb, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731564984823 2024-11-14T06:16:27,325 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c720d0ece964299bf31b9016dd94247, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731564985180 2024-11-14T06:16:27,326 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting aad6656df5da42168acb9eef03242cf8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731564985180 2024-11-14T06:16:27,326 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e0435bdfcd043e89bea854a2d7a4110, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1731564985904 2024-11-14T06:16:27,327 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2165039abefd4e1f993da62a4f89bb44, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1731564985904 2024-11-14T06:16:27,327 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 742db572e4a3425bb1cd2d97ab808016, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731564986586 2024-11-14T06:16:27,328 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 3242feb0dd8943e5bdc5e89efbf3bc6c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731564986586 2024-11-14T06:16:27,364 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7851abd86d644f97c54179c322b1c1e#A#compaction#82 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:16:27,365 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7851abd86d644f97c54179c322b1c1e#B#compaction#81 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:16:27,365 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/3aedc79178344f0cb15dc520dbab081e is 50, key is test_row_0/A:col10/1731564986699/Put/seqid=0 2024-11-14T06:16:27,365 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/1788cd6470724dd599006d151f3487da is 50, key is test_row_0/B:col10/1731564986699/Put/seqid=0 2024-11-14T06:16:27,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741918_1094 (size=13221) 2024-11-14T06:16:27,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741919_1095 (size=13221) 2024-11-14T06:16:27,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on d7851abd86d644f97c54179c322b1c1e 2024-11-14T06:16:27,391 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d7851abd86d644f97c54179c322b1c1e 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-14T06:16:27,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=A 2024-11-14T06:16:27,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:27,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=B 2024-11-14T06:16:27,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:27,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=C 2024-11-14T06:16:27,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:27,396 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/1788cd6470724dd599006d151f3487da as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/1788cd6470724dd599006d151f3487da 2024-11-14T06:16:27,396 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/3aedc79178344f0cb15dc520dbab081e as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/3aedc79178344f0cb15dc520dbab081e 2024-11-14T06:16:27,399 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/d27b070d66654bae9c5539b3da3bbb18 is 50, key is test_row_0/A:col10/1731564987389/Put/seqid=0 2024-11-14T06:16:27,408 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d7851abd86d644f97c54179c322b1c1e/A of d7851abd86d644f97c54179c322b1c1e into 3aedc79178344f0cb15dc520dbab081e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:16:27,408 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7851abd86d644f97c54179c322b1c1e: 2024-11-14T06:16:27,408 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., storeName=d7851abd86d644f97c54179c322b1c1e/A, priority=12, startTime=1731564987318; duration=0sec 2024-11-14T06:16:27,408 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:16:27,408 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7851abd86d644f97c54179c322b1c1e:A 2024-11-14T06:16:27,409 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T06:16:27,410 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d7851abd86d644f97c54179c322b1c1e/B of d7851abd86d644f97c54179c322b1c1e into 1788cd6470724dd599006d151f3487da(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:16:27,410 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7851abd86d644f97c54179c322b1c1e: 2024-11-14T06:16:27,410 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., storeName=d7851abd86d644f97c54179c322b1c1e/B, priority=12, startTime=1731564987318; duration=0sec 2024-11-14T06:16:27,410 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:16:27,410 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7851abd86d644f97c54179c322b1c1e:B 2024-11-14T06:16:27,410 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T06:16:27,410 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): d7851abd86d644f97c54179c322b1c1e/C is initiating minor compaction (all files) 2024-11-14T06:16:27,410 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7851abd86d644f97c54179c322b1c1e/C in TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:27,411 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/3074e07bef3f4ba39489fae3dd75c843, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/bc9e694409584947aaea92a3c1e01e8d, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/c3fa25b1cea94818bfe48c384f3051fc, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/4cf065cc93284835a08c11ff91ab262f] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp, totalSize=48.8 K 2024-11-14T06:16:27,411 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3074e07bef3f4ba39489fae3dd75c843, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731564984823 2024-11-14T06:16:27,412 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc9e694409584947aaea92a3c1e01e8d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731564985180 2024-11-14T06:16:27,412 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3fa25b1cea94818bfe48c384f3051fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1731564985904 2024-11-14T06:16:27,413 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4cf065cc93284835a08c11ff91ab262f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731564986586 2024-11-14T06:16:27,476 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7851abd86d644f97c54179c322b1c1e#C#compaction#84 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:16:27,476 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/f82a1de4495548318341f0f8da0c9673 is 50, key is test_row_0/C:col10/1731564986699/Put/seqid=0 2024-11-14T06:16:27,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:27,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565047456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:27,478 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:27,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:27,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:27,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741920_1096 (size=14741) 2024-11-14T06:16:27,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:27,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565047473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:27,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:27,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:27,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565047474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:27,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:27,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:27,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:27,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:27,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:27,485 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/d27b070d66654bae9c5539b3da3bbb18 2024-11-14T06:16:27,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:27,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565047478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:27,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:27,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:27,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:27,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:27,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565047481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:27,502 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:27,502 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:27,502 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:27,511 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/1715a6cca5b643f3be5f0473471a93f6 is 50, key is test_row_0/B:col10/1731564987389/Put/seqid=0 2024-11-14T06:16:27,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741921_1097 (size=13221) 2024-11-14T06:16:27,531 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/f82a1de4495548318341f0f8da0c9673 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/f82a1de4495548318341f0f8da0c9673 2024-11-14T06:16:27,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741922_1098 (size=12301) 2024-11-14T06:16:27,534 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/1715a6cca5b643f3be5f0473471a93f6 2024-11-14T06:16:27,545 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d7851abd86d644f97c54179c322b1c1e/C of d7851abd86d644f97c54179c322b1c1e into f82a1de4495548318341f0f8da0c9673(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:16:27,545 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7851abd86d644f97c54179c322b1c1e: 2024-11-14T06:16:27,545 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., storeName=d7851abd86d644f97c54179c322b1c1e/C, priority=12, startTime=1731564987318; duration=0sec 2024-11-14T06:16:27,545 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:16:27,545 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7851abd86d644f97c54179c322b1c1e:C 2024-11-14T06:16:27,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-14T06:16:27,548 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:16:27,549 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T06:16:27,550 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/e15e4ef541da4d3580e0400b8215a519 is 50, key is test_row_0/C:col10/1731564987389/Put/seqid=0 2024-11-14T06:16:27,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-14T06:16:27,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T06:16:27,554 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T06:16:27,556 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T06:16:27,556 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T06:16:27,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:27,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:27,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565047593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:27,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:27,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:27,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:27,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:27,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565047596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:27,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:27,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:27,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:27,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565047594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:27,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:27,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:27,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:27,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741923_1099 (size=12301) 2024-11-14T06:16:27,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:27,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565047608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:27,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:27,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565047600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:27,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:27,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:27,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:27,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:27,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:27,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:27,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T06:16:27,714 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-14T06:16:27,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:27,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. as already flushing 2024-11-14T06:16:27,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:27,715 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:27,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:27,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:27,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:27,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565047808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:27,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:27,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565047808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:27,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:27,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:27,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:27,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:27,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565047808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:27,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:27,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:27,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:27,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:27,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:27,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:27,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:27,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:27,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565047818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:27,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:27,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:27,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:27,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565047818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:27,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:27,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:27,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:27,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T06:16:27,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-14T06:16:27,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:27,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. as already flushing 2024-11-14T06:16:27,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:27,872 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:27,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:27,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:28,006 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/e15e4ef541da4d3580e0400b8215a519 2024-11-14T06:16:28,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-14T06:16:28,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:28,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. as already flushing 2024-11-14T06:16:28,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:28,027 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:28,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:28,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:28,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/d27b070d66654bae9c5539b3da3bbb18 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/d27b070d66654bae9c5539b3da3bbb18 2024-11-14T06:16:28,041 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/d27b070d66654bae9c5539b3da3bbb18, entries=200, sequenceid=418, filesize=14.4 K 2024-11-14T06:16:28,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/1715a6cca5b643f3be5f0473471a93f6 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/1715a6cca5b643f3be5f0473471a93f6 2024-11-14T06:16:28,063 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/1715a6cca5b643f3be5f0473471a93f6, entries=150, sequenceid=418, filesize=12.0 K 2024-11-14T06:16:28,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/e15e4ef541da4d3580e0400b8215a519 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/e15e4ef541da4d3580e0400b8215a519 2024-11-14T06:16:28,075 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/e15e4ef541da4d3580e0400b8215a519, entries=150, sequenceid=418, filesize=12.0 K 2024-11-14T06:16:28,076 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for d7851abd86d644f97c54179c322b1c1e in 685ms, sequenceid=418, compaction requested=false 2024-11-14T06:16:28,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d7851abd86d644f97c54179c322b1c1e: 2024-11-14T06:16:28,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on d7851abd86d644f97c54179c322b1c1e 2024-11-14T06:16:28,123 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d7851abd86d644f97c54179c322b1c1e 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-14T06:16:28,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=A 2024-11-14T06:16:28,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:28,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=B 2024-11-14T06:16:28,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:28,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=C 2024-11-14T06:16:28,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:28,145 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/d8b54246acc0450f9f707be0a92cf9d9 is 50, key is test_row_0/A:col10/1731564988121/Put/seqid=0 2024-11-14T06:16:28,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:28,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565048154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:28,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:28,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565048154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:28,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:28,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:28,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:28,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:28,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:28,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:28,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:28,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565048159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:28,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:28,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:28,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:28,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:28,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565048161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:28,174 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:28,174 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:28,174 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:28,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T06:16:28,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:28,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565048165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:28,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:28,181 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-14T06:16:28,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:28,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:28,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:28,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. as already flushing 2024-11-14T06:16:28,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:28,182 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:28,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:28,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:28,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741924_1100 (size=14741) 2024-11-14T06:16:28,221 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/d8b54246acc0450f9f707be0a92cf9d9 2024-11-14T06:16:28,245 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/5803fae03a8f4e5ead99d48b7692b186 is 50, key is test_row_0/B:col10/1731564988121/Put/seqid=0 2024-11-14T06:16:28,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741925_1101 (size=12301) 2024-11-14T06:16:28,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:28,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565048278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:28,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:28,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565048278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:28,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:28,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565048278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:28,287 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:28,287 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:28,287 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:28,288 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:28,288 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:28,288 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:28,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:28,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:28,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:28,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:28,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565048289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:28,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:28,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:28,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:28,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:28,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565048278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:28,307 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:28,307 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:28,307 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:28,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-14T06:16:28,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:28,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. as already flushing 2024-11-14T06:16:28,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:28,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:28,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:28,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:28,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-14T06:16:28,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:28,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. as already flushing 2024-11-14T06:16:28,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:28,495 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:28,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:28,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:28,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:28,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565048504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:28,513 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:28,513 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:28,513 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:28,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:28,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565048504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:28,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:28,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:28,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:28,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:28,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565048505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:28,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:28,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565048504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:28,516 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:28,516 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:28,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:28,516 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:28,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:28,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:28,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:28,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565048517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:28,531 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:28,531 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:28,531 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:28,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-14T06:16:28,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:28,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. as already flushing 2024-11-14T06:16:28,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:28,652 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:28,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:28,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:28,655 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/5803fae03a8f4e5ead99d48b7692b186 2024-11-14T06:16:28,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T06:16:28,688 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/f5dab5c14e2248cc8f4eeb4a407072dd is 50, key is test_row_0/C:col10/1731564988121/Put/seqid=0 2024-11-14T06:16:28,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741926_1102 (size=12301) 2024-11-14T06:16:28,808 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-14T06:16:28,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:28,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. as already flushing 2024-11-14T06:16:28,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:28,812 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:28,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:28,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:28,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:28,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565048817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:28,830 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:28,830 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:28,830 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:28,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:28,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565048833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:28,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:28,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:28,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:28,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565048833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:28,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:28,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:28,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:28,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:28,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:28,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565048834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:28,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:28,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:28,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:28,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:28,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565048837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:28,843 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:28,843 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:28,843 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:28,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-14T06:16:28,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:28,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. as already flushing 2024-11-14T06:16:28,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:28,966 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:28,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:28,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:29,121 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-14T06:16:29,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:29,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. as already flushing 2024-11-14T06:16:29,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:29,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:29,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:29,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:29,155 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/f5dab5c14e2248cc8f4eeb4a407072dd 2024-11-14T06:16:29,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/d8b54246acc0450f9f707be0a92cf9d9 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/d8b54246acc0450f9f707be0a92cf9d9 2024-11-14T06:16:29,208 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/d8b54246acc0450f9f707be0a92cf9d9, entries=200, sequenceid=439, filesize=14.4 K 2024-11-14T06:16:29,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/5803fae03a8f4e5ead99d48b7692b186 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/5803fae03a8f4e5ead99d48b7692b186 2024-11-14T06:16:29,238 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/5803fae03a8f4e5ead99d48b7692b186, entries=150, sequenceid=439, filesize=12.0 K 2024-11-14T06:16:29,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/f5dab5c14e2248cc8f4eeb4a407072dd as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/f5dab5c14e2248cc8f4eeb4a407072dd 2024-11-14T06:16:29,247 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/f5dab5c14e2248cc8f4eeb4a407072dd, entries=150, sequenceid=439, filesize=12.0 K 2024-11-14T06:16:29,248 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for d7851abd86d644f97c54179c322b1c1e in 1125ms, sequenceid=439, compaction requested=true 2024-11-14T06:16:29,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d7851abd86d644f97c54179c322b1c1e: 2024-11-14T06:16:29,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7851abd86d644f97c54179c322b1c1e:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T06:16:29,249 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:16:29,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:16:29,249 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:16:29,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7851abd86d644f97c54179c322b1c1e:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T06:16:29,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:16:29,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7851abd86d644f97c54179c322b1c1e:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T06:16:29,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:16:29,250 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:16:29,250 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42703 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:16:29,250 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): d7851abd86d644f97c54179c322b1c1e/B is initiating minor compaction (all files) 2024-11-14T06:16:29,250 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): d7851abd86d644f97c54179c322b1c1e/A is initiating minor compaction (all files) 2024-11-14T06:16:29,250 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7851abd86d644f97c54179c322b1c1e/B in TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:29,250 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7851abd86d644f97c54179c322b1c1e/A in TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:29,250 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/1788cd6470724dd599006d151f3487da, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/1715a6cca5b643f3be5f0473471a93f6, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/5803fae03a8f4e5ead99d48b7692b186] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp, totalSize=36.9 K 2024-11-14T06:16:29,250 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/3aedc79178344f0cb15dc520dbab081e, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/d27b070d66654bae9c5539b3da3bbb18, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/d8b54246acc0450f9f707be0a92cf9d9] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp, totalSize=41.7 K 2024-11-14T06:16:29,251 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 1788cd6470724dd599006d151f3487da, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731564986586 2024-11-14T06:16:29,251 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3aedc79178344f0cb15dc520dbab081e, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731564986586 2024-11-14T06:16:29,251 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 1715a6cca5b643f3be5f0473471a93f6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1731564986742 2024-11-14T06:16:29,251 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting d27b070d66654bae9c5539b3da3bbb18, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1731564986737 2024-11-14T06:16:29,252 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 5803fae03a8f4e5ead99d48b7692b186, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1731564987472 2024-11-14T06:16:29,252 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8b54246acc0450f9f707be0a92cf9d9, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1731564987449 2024-11-14T06:16:29,263 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7851abd86d644f97c54179c322b1c1e#A#compaction#90 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:16:29,263 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/59b11410b705450daae5f3dd49736739 is 50, key is test_row_0/A:col10/1731564988121/Put/seqid=0 2024-11-14T06:16:29,266 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7851abd86d644f97c54179c322b1c1e#B#compaction#91 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:16:29,266 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/5d5c93fd45504cbbb3a00c96f78f3d4f is 50, key is test_row_0/B:col10/1731564988121/Put/seqid=0 2024-11-14T06:16:29,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-14T06:16:29,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:29,282 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing d7851abd86d644f97c54179c322b1c1e 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-14T06:16:29,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=A 2024-11-14T06:16:29,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:29,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=B 2024-11-14T06:16:29,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:29,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=C 2024-11-14T06:16:29,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:29,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741927_1103 (size=13323) 2024-11-14T06:16:29,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/ab801cb7ada24b73ba7759e7c4f09f5b is 50, key is test_row_0/A:col10/1731564988153/Put/seqid=0 2024-11-14T06:16:29,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741928_1104 (size=13323) 2024-11-14T06:16:29,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741929_1105 (size=12301) 2024-11-14T06:16:29,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. as already flushing 2024-11-14T06:16:29,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on d7851abd86d644f97c54179c322b1c1e 2024-11-14T06:16:29,370 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/ab801cb7ada24b73ba7759e7c4f09f5b 2024-11-14T06:16:29,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/c4a05ae4a16a45a08536fbf1ac21b27f is 50, key is test_row_0/B:col10/1731564988153/Put/seqid=0 2024-11-14T06:16:29,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:29,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565049412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:29,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:29,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:29,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:29,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:29,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565049416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:29,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:29,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:29,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:29,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:29,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565049418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:29,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:29,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:29,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:29,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:29,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565049418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:29,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:29,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:29,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:29,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:29,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565049426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:29,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:29,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:29,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:29,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741930_1106 (size=12301) 2024-11-14T06:16:29,475 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/c4a05ae4a16a45a08536fbf1ac21b27f 2024-11-14T06:16:29,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/575f6a6bbb294460859aa8a926982506 is 50, key is test_row_0/C:col10/1731564988153/Put/seqid=0 2024-11-14T06:16:29,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741931_1107 (size=12301) 2024-11-14T06:16:29,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:29,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565049538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:29,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:29,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565049538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:29,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:29,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:29,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:29,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:29,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565049538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:29,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:29,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:29,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:29,553 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:29,553 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:29,553 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:29,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:29,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565049547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:29,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:29,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:29,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:29,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:29,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565049558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:29,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:29,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:29,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:29,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T06:16:29,728 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/59b11410b705450daae5f3dd49736739 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/59b11410b705450daae5f3dd49736739 2024-11-14T06:16:29,752 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/5d5c93fd45504cbbb3a00c96f78f3d4f as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/5d5c93fd45504cbbb3a00c96f78f3d4f 2024-11-14T06:16:29,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:29,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565049757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:29,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:29,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:29,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:29,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:29,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565049757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:29,763 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:29,763 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:29,763 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:29,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:29,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565049758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:29,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:29,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:29,767 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d7851abd86d644f97c54179c322b1c1e/B of d7851abd86d644f97c54179c322b1c1e into 5d5c93fd45504cbbb3a00c96f78f3d4f(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:16:29,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:29,767 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7851abd86d644f97c54179c322b1c1e: 2024-11-14T06:16:29,767 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., storeName=d7851abd86d644f97c54179c322b1c1e/B, priority=13, startTime=1731564989249; duration=0sec 2024-11-14T06:16:29,768 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:16:29,768 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7851abd86d644f97c54179c322b1c1e:B 2024-11-14T06:16:29,768 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:16:29,770 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d7851abd86d644f97c54179c322b1c1e/A of d7851abd86d644f97c54179c322b1c1e into 59b11410b705450daae5f3dd49736739(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:16:29,770 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7851abd86d644f97c54179c322b1c1e: 2024-11-14T06:16:29,771 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., storeName=d7851abd86d644f97c54179c322b1c1e/A, priority=13, startTime=1731564989249; duration=0sec 2024-11-14T06:16:29,771 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:16:29,771 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7851abd86d644f97c54179c322b1c1e:A 2024-11-14T06:16:29,779 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:16:29,780 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): d7851abd86d644f97c54179c322b1c1e/C is initiating minor compaction (all files) 2024-11-14T06:16:29,780 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7851abd86d644f97c54179c322b1c1e/C in TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:29,780 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/f82a1de4495548318341f0f8da0c9673, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/e15e4ef541da4d3580e0400b8215a519, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/f5dab5c14e2248cc8f4eeb4a407072dd] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp, totalSize=36.9 K 2024-11-14T06:16:29,785 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting f82a1de4495548318341f0f8da0c9673, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731564986586 2024-11-14T06:16:29,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:29,788 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting e15e4ef541da4d3580e0400b8215a519, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1731564986742 2024-11-14T06:16:29,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565049765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:29,796 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:29,796 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:29,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:29,796 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:29,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565049768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:29,796 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting f5dab5c14e2248cc8f4eeb4a407072dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1731564987472 2024-11-14T06:16:29,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:29,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:29,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:29,842 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7851abd86d644f97c54179c322b1c1e#C#compaction#95 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:16:29,843 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/dde1275ca2ac4f398128daddf3a25849 is 50, key is test_row_0/C:col10/1731564988121/Put/seqid=0 2024-11-14T06:16:29,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741932_1108 (size=13323) 2024-11-14T06:16:29,941 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/575f6a6bbb294460859aa8a926982506 2024-11-14T06:16:29,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/ab801cb7ada24b73ba7759e7c4f09f5b as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/ab801cb7ada24b73ba7759e7c4f09f5b 2024-11-14T06:16:29,962 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/ab801cb7ada24b73ba7759e7c4f09f5b, entries=150, sequenceid=453, filesize=12.0 K 2024-11-14T06:16:29,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/c4a05ae4a16a45a08536fbf1ac21b27f as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/c4a05ae4a16a45a08536fbf1ac21b27f 2024-11-14T06:16:29,971 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/c4a05ae4a16a45a08536fbf1ac21b27f, entries=150, sequenceid=453, filesize=12.0 K 2024-11-14T06:16:29,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/575f6a6bbb294460859aa8a926982506 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/575f6a6bbb294460859aa8a926982506 2024-11-14T06:16:29,980 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/575f6a6bbb294460859aa8a926982506, entries=150, sequenceid=453, filesize=12.0 K 2024-11-14T06:16:29,981 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for d7851abd86d644f97c54179c322b1c1e in 699ms, sequenceid=453, compaction requested=false 2024-11-14T06:16:29,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for d7851abd86d644f97c54179c322b1c1e: 2024-11-14T06:16:29,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:29,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-14T06:16:29,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-14T06:16:29,989 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-14T06:16:29,990 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4310 sec 2024-11-14T06:16:29,993 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 2.4420 sec 2024-11-14T06:16:30,083 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d7851abd86d644f97c54179c322b1c1e 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-14T06:16:30,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on d7851abd86d644f97c54179c322b1c1e 2024-11-14T06:16:30,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=A 2024-11-14T06:16:30,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:30,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=B 2024-11-14T06:16:30,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:30,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=C 2024-11-14T06:16:30,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:30,109 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/1f2a1b788cc9483391c0b77ba500d125 is 50, key is test_row_0/A:col10/1731564990072/Put/seqid=0 2024-11-14T06:16:30,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:30,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565050114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:30,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:30,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:30,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:30,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:30,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565050121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:30,137 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:30,137 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:30,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:30,137 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:30,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:30,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565050124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:30,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:30,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565050127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:30,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:30,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:30,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:30,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:30,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:30,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:30,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565050121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:30,146 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:30,146 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:30,146 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:30,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741933_1109 (size=14737) 2024-11-14T06:16:30,165 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=475 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/1f2a1b788cc9483391c0b77ba500d125 2024-11-14T06:16:30,216 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/93430d58e79749c9a5091c8805b0be83 is 50, key is test_row_0/B:col10/1731564990072/Put/seqid=0 2024-11-14T06:16:30,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:30,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:30,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565050244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:30,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565050248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:30,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:30,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:30,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:30,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:30,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:30,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:30,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:30,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565050252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:30,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741934_1110 (size=9857) 2024-11-14T06:16:30,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:30,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:30,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:30,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:30,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565050253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:30,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:30,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:30,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:30,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:30,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565050259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:30,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:30,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:30,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:30,301 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/dde1275ca2ac4f398128daddf3a25849 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/dde1275ca2ac4f398128daddf3a25849 2024-11-14T06:16:30,317 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d7851abd86d644f97c54179c322b1c1e/C of d7851abd86d644f97c54179c322b1c1e into dde1275ca2ac4f398128daddf3a25849(size=13.0 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:16:30,317 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7851abd86d644f97c54179c322b1c1e: 2024-11-14T06:16:30,317 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., storeName=d7851abd86d644f97c54179c322b1c1e/C, priority=13, startTime=1731564989249; duration=0sec 2024-11-14T06:16:30,317 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:16:30,317 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7851abd86d644f97c54179c322b1c1e:C 2024-11-14T06:16:30,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:30,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565050468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:30,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:30,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:30,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:30,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:30,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565050468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:30,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:30,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:30,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:30,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:30,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565050478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:30,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:30,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:30,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:30,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:30,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565050478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:30,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:30,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:30,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:30,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:30,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565050478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:30,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:30,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:30,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:30,674 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=475 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/93430d58e79749c9a5091c8805b0be83 2024-11-14T06:16:30,686 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/36b9566d40e145b4889fec1f5826fc37 is 50, key is test_row_0/C:col10/1731564990072/Put/seqid=0 2024-11-14T06:16:30,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741935_1111 (size=9857) 2024-11-14T06:16:30,710 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=475 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/36b9566d40e145b4889fec1f5826fc37 2024-11-14T06:16:30,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/1f2a1b788cc9483391c0b77ba500d125 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/1f2a1b788cc9483391c0b77ba500d125 2024-11-14T06:16:30,733 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/1f2a1b788cc9483391c0b77ba500d125, entries=200, sequenceid=475, filesize=14.4 K 2024-11-14T06:16:30,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/93430d58e79749c9a5091c8805b0be83 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/93430d58e79749c9a5091c8805b0be83 2024-11-14T06:16:30,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/93430d58e79749c9a5091c8805b0be83, entries=100, sequenceid=475, filesize=9.6 K 2024-11-14T06:16:30,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/36b9566d40e145b4889fec1f5826fc37 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/36b9566d40e145b4889fec1f5826fc37 2024-11-14T06:16:30,758 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/36b9566d40e145b4889fec1f5826fc37, entries=100, sequenceid=475, filesize=9.6 K 2024-11-14T06:16:30,759 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for d7851abd86d644f97c54179c322b1c1e in 676ms, sequenceid=475, compaction requested=true 2024-11-14T06:16:30,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d7851abd86d644f97c54179c322b1c1e: 2024-11-14T06:16:30,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7851abd86d644f97c54179c322b1c1e:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T06:16:30,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:16:30,760 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:16:30,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7851abd86d644f97c54179c322b1c1e:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T06:16:30,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:16:30,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7851abd86d644f97c54179c322b1c1e:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T06:16:30,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T06:16:30,761 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:16:30,762 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40361 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:16:30,762 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): d7851abd86d644f97c54179c322b1c1e/A is initiating minor compaction (all files) 2024-11-14T06:16:30,762 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7851abd86d644f97c54179c322b1c1e/A in TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:30,762 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/59b11410b705450daae5f3dd49736739, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/ab801cb7ada24b73ba7759e7c4f09f5b, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/1f2a1b788cc9483391c0b77ba500d125] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp, totalSize=39.4 K 2024-11-14T06:16:30,762 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35481 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:16:30,763 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): d7851abd86d644f97c54179c322b1c1e/B is initiating minor compaction (all files) 2024-11-14T06:16:30,763 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7851abd86d644f97c54179c322b1c1e/B in TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:30,763 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/5d5c93fd45504cbbb3a00c96f78f3d4f, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/c4a05ae4a16a45a08536fbf1ac21b27f, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/93430d58e79749c9a5091c8805b0be83] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp, totalSize=34.6 K 2024-11-14T06:16:30,763 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 59b11410b705450daae5f3dd49736739, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1731564987472 2024-11-14T06:16:30,763 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d5c93fd45504cbbb3a00c96f78f3d4f, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1731564987472 2024-11-14T06:16:30,763 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting ab801cb7ada24b73ba7759e7c4f09f5b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1731564988153 2024-11-14T06:16:30,763 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4a05ae4a16a45a08536fbf1ac21b27f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1731564988153 2024-11-14T06:16:30,764 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f2a1b788cc9483391c0b77ba500d125, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=475, earliestPutTs=1731564989415 2024-11-14T06:16:30,764 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93430d58e79749c9a5091c8805b0be83, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=475, earliestPutTs=1731564990072 2024-11-14T06:16:30,779 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7851abd86d644f97c54179c322b1c1e#B#compaction#99 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:16:30,779 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d7851abd86d644f97c54179c322b1c1e 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-14T06:16:30,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=A 2024-11-14T06:16:30,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:30,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=B 2024-11-14T06:16:30,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:30,780 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/a57658a5458645b8a17f839ea011b1e1 is 50, key is test_row_0/B:col10/1731564990072/Put/seqid=0 2024-11-14T06:16:30,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=C 2024-11-14T06:16:30,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:30,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on d7851abd86d644f97c54179c322b1c1e 2024-11-14T06:16:30,808 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/87511dfe2fe3404da250f090eddb3d1d is 50, key is test_row_0/A:col10/1731564990115/Put/seqid=0 2024-11-14T06:16:30,814 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7851abd86d644f97c54179c322b1c1e#A#compaction#101 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:16:30,815 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/748a896e4404465fb3541699a6c4e0b9 is 50, key is test_row_0/A:col10/1731564990072/Put/seqid=0 2024-11-14T06:16:30,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741936_1112 (size=13425) 2024-11-14T06:16:30,836 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/a57658a5458645b8a17f839ea011b1e1 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/a57658a5458645b8a17f839ea011b1e1 2024-11-14T06:16:30,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:30,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565050821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:30,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:30,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565050822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:30,841 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:30,841 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:30,841 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:30,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:30,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565050824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:30,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:30,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:30,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:30,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:30,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565050824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:30,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:30,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:30,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:30,844 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:30,844 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:30,844 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:30,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:30,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565050839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:30,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:30,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:30,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:30,866 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d7851abd86d644f97c54179c322b1c1e/B of d7851abd86d644f97c54179c322b1c1e into a57658a5458645b8a17f839ea011b1e1(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:16:30,866 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7851abd86d644f97c54179c322b1c1e: 2024-11-14T06:16:30,866 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., storeName=d7851abd86d644f97c54179c322b1c1e/B, priority=13, startTime=1731564990760; duration=0sec 2024-11-14T06:16:30,867 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:16:30,867 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7851abd86d644f97c54179c322b1c1e:B 2024-11-14T06:16:30,867 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:16:30,871 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35481 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:16:30,871 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): d7851abd86d644f97c54179c322b1c1e/C is initiating minor compaction (all files) 2024-11-14T06:16:30,871 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7851abd86d644f97c54179c322b1c1e/C in TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:30,871 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/dde1275ca2ac4f398128daddf3a25849, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/575f6a6bbb294460859aa8a926982506, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/36b9566d40e145b4889fec1f5826fc37] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp, totalSize=34.6 K 2024-11-14T06:16:30,872 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting dde1275ca2ac4f398128daddf3a25849, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1731564987472 2024-11-14T06:16:30,872 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 575f6a6bbb294460859aa8a926982506, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1731564988153 2024-11-14T06:16:30,881 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36b9566d40e145b4889fec1f5826fc37, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=475, earliestPutTs=1731564990072 2024-11-14T06:16:30,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741937_1113 (size=14741) 2024-11-14T06:16:30,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741938_1114 (size=13425) 2024-11-14T06:16:30,902 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7851abd86d644f97c54179c322b1c1e#C#compaction#102 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:16:30,903 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/42c6ada6cfe048dc91560b39f5794930 is 50, key is test_row_0/C:col10/1731564990072/Put/seqid=0 2024-11-14T06:16:30,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741939_1115 (size=13425) 2024-11-14T06:16:30,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:30,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565050949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:30,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:30,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:30,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:30,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:30,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565050949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:30,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:30,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565050949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:30,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:30,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:30,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:30,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:30,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:30,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:30,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:30,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565050952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:30,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:30,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565050959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:30,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:30,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:30,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:30,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:30,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:30,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:30,999 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/42c6ada6cfe048dc91560b39f5794930 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/42c6ada6cfe048dc91560b39f5794930 2024-11-14T06:16:31,014 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d7851abd86d644f97c54179c322b1c1e/C of d7851abd86d644f97c54179c322b1c1e into 42c6ada6cfe048dc91560b39f5794930(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:16:31,015 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7851abd86d644f97c54179c322b1c1e: 2024-11-14T06:16:31,015 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., storeName=d7851abd86d644f97c54179c322b1c1e/C, priority=13, startTime=1731564990760; duration=0sec 2024-11-14T06:16:31,015 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:16:31,015 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7851abd86d644f97c54179c322b1c1e:C 2024-11-14T06:16:31,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:31,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565051161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:31,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:31,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:31,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:31,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:31,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565051169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:31,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:31,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:31,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:31,176 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:31,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565051169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:31,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:31,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565051173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:31,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:31,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:31,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:31,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:31,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565051178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:31,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:31,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:31,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:31,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:31,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:31,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:31,284 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=489 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/87511dfe2fe3404da250f090eddb3d1d 2024-11-14T06:16:31,294 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/748a896e4404465fb3541699a6c4e0b9 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/748a896e4404465fb3541699a6c4e0b9 2024-11-14T06:16:31,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/98940bed0fd443dca349244266a183db is 50, key is test_row_0/B:col10/1731564990115/Put/seqid=0 2024-11-14T06:16:31,322 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d7851abd86d644f97c54179c322b1c1e/A of d7851abd86d644f97c54179c322b1c1e into 748a896e4404465fb3541699a6c4e0b9(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:16:31,323 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7851abd86d644f97c54179c322b1c1e: 2024-11-14T06:16:31,323 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., storeName=d7851abd86d644f97c54179c322b1c1e/A, priority=13, startTime=1731564990759; duration=0sec 2024-11-14T06:16:31,323 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:16:31,323 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7851abd86d644f97c54179c322b1c1e:A 2024-11-14T06:16:31,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741940_1116 (size=12301) 2024-11-14T06:16:31,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:31,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565051478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:31,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:31,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:31,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:31,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:31,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565051487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:31,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:31,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:31,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:31,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:31,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565051487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:31,492 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:31,492 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:31,492 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:31,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:31,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565051487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:31,499 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:31,499 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:31,499 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:31,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:31,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565051512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:31,518 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:31,518 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:31,518 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:31,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T06:16:31,718 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:16:31,719 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T06:16:31,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-14T06:16:31,725 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T06:16:31,732 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T06:16:31,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-14T06:16:31,733 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T06:16:31,768 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=489 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/98940bed0fd443dca349244266a183db 2024-11-14T06:16:31,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/637ad1cdf64248ea872d9d33614071bc is 50, key is test_row_0/C:col10/1731564990115/Put/seqid=0 2024-11-14T06:16:31,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-14T06:16:31,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741941_1117 (size=12301) 2024-11-14T06:16:31,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-14T06:16:31,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:31,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. as already flushing 2024-11-14T06:16:31,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:31,889 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:31,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:31,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:31,927 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-14T06:16:31,927 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T06:16:31,935 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-14T06:16:31,935 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T06:16:31,937 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:16:31,938 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:16:31,938 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:16:31,938 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:16:31,939 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:16:31,939 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-14T06:16:31,939 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T06:16:31,939 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:16:31,939 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:16:31,939 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:16:31,940 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:16:31,947 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-14T06:16:31,947 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T06:16:31,947 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:16:31,947 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:16:31,952 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:16:31,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:31,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58308 deadline: 1731565051998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:32,000 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:32,000 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:32,000 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:32,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:32,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:32,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58350 deadline: 1731565051998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:32,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58292 deadline: 1731565051998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:32,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:32,002 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:32,002 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:32,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:32,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:32,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:32,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:32,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58366 deadline: 1731565052008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:32,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:32,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:32,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:32,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:32,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58294 deadline: 1731565052028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:32,030 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:32,030 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:32,030 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:16:32,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-14T06:16:32,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:32,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. as already flushing 2024-11-14T06:16:32,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:32,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:32,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:32,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:32,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-14T06:16:32,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-14T06:16:32,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:32,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. as already flushing 2024-11-14T06:16:32,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:32,201 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:32,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:32,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:32,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-14T06:16:32,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:32,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. as already flushing 2024-11-14T06:16:32,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:32,354 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:32,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:32,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:32,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-14T06:16:32,379 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=489 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/637ad1cdf64248ea872d9d33614071bc 2024-11-14T06:16:32,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/87511dfe2fe3404da250f090eddb3d1d as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/87511dfe2fe3404da250f090eddb3d1d 2024-11-14T06:16:32,507 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/87511dfe2fe3404da250f090eddb3d1d, entries=200, sequenceid=489, filesize=14.4 K 2024-11-14T06:16:32,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/98940bed0fd443dca349244266a183db as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/98940bed0fd443dca349244266a183db 2024-11-14T06:16:32,514 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/98940bed0fd443dca349244266a183db, entries=150, sequenceid=489, filesize=12.0 K 2024-11-14T06:16:32,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/637ad1cdf64248ea872d9d33614071bc as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/637ad1cdf64248ea872d9d33614071bc 2024-11-14T06:16:32,522 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/637ad1cdf64248ea872d9d33614071bc, entries=150, sequenceid=489, filesize=12.0 K 2024-11-14T06:16:32,525 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for d7851abd86d644f97c54179c322b1c1e in 1746ms, sequenceid=489, compaction requested=false 2024-11-14T06:16:32,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d7851abd86d644f97c54179c322b1c1e: 2024-11-14T06:16:32,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-14T06:16:32,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:32,531 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing d7851abd86d644f97c54179c322b1c1e 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-14T06:16:32,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=A 2024-11-14T06:16:32,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:32,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=B 2024-11-14T06:16:32,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:32,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=C 2024-11-14T06:16:32,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:32,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/09f394e03b72477c984387486680f0dc is 50, key is test_row_0/A:col10/1731564990836/Put/seqid=0 2024-11-14T06:16:32,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741942_1118 (size=12301) 2024-11-14T06:16:32,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-14T06:16:32,943 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=511 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/09f394e03b72477c984387486680f0dc 2024-11-14T06:16:32,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/f3aee24b80754aae8998a26e5e114b31 is 50, key is test_row_0/B:col10/1731564990836/Put/seqid=0 2024-11-14T06:16:32,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741943_1119 (size=12301) 2024-11-14T06:16:33,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on d7851abd86d644f97c54179c322b1c1e 2024-11-14T06:16:33,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. as already flushing 2024-11-14T06:16:33,009 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-14T06:16:33,009 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-14T06:16:33,009 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T06:16:33,009 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T06:16:33,009 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:16:33,009 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:16:33,009 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:16:33,009 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:16:33,009 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:16:33,009 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:16:33,019 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-14T06:16:33,019 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-14T06:16:33,019 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T06:16:33,019 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T06:16:33,019 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:16:33,019 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:16:33,020 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:16:33,020 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:16:33,020 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:16:33,020 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:16:33,039 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-14T06:16:33,039 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T06:16:33,039 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:16:33,039 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:16:33,040 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:16:33,355 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=511 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/f3aee24b80754aae8998a26e5e114b31 2024-11-14T06:16:33,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/9d15e92ba0da41bebb83fce36483d4ab is 50, key is test_row_0/C:col10/1731564990836/Put/seqid=0 2024-11-14T06:16:33,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741944_1120 (size=12301) 2024-11-14T06:16:33,767 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=511 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/9d15e92ba0da41bebb83fce36483d4ab 2024-11-14T06:16:33,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/09f394e03b72477c984387486680f0dc as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/09f394e03b72477c984387486680f0dc 2024-11-14T06:16:33,783 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/09f394e03b72477c984387486680f0dc, entries=150, sequenceid=511, filesize=12.0 K 2024-11-14T06:16:33,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/f3aee24b80754aae8998a26e5e114b31 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/f3aee24b80754aae8998a26e5e114b31 2024-11-14T06:16:33,797 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/f3aee24b80754aae8998a26e5e114b31, entries=150, sequenceid=511, filesize=12.0 K 2024-11-14T06:16:33,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/9d15e92ba0da41bebb83fce36483d4ab as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/9d15e92ba0da41bebb83fce36483d4ab 2024-11-14T06:16:33,821 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/9d15e92ba0da41bebb83fce36483d4ab, entries=150, sequenceid=511, filesize=12.0 K 2024-11-14T06:16:33,830 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=33.54 KB/34350 for d7851abd86d644f97c54179c322b1c1e in 1300ms, sequenceid=511, compaction requested=true 2024-11-14T06:16:33,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for d7851abd86d644f97c54179c322b1c1e: 2024-11-14T06:16:33,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:33,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-14T06:16:33,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-14T06:16:33,842 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-14T06:16:33,842 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1060 sec 2024-11-14T06:16:33,845 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 2.1240 sec 2024-11-14T06:16:33,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-14T06:16:33,878 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:16:33,879 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T06:16:33,879 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 79 2024-11-14T06:16:33,879 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2024-11-14T06:16:33,879 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 80 2024-11-14T06:16:33,879 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 78 2024-11-14T06:16:33,879 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 83 2024-11-14T06:16:33,879 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T06:16:33,879 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1462 2024-11-14T06:16:33,879 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1464 2024-11-14T06:16:33,879 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T06:16:33,879 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 705 2024-11-14T06:16:33,879 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2113 rows 2024-11-14T06:16:33,879 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 716 2024-11-14T06:16:33,879 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2143 rows 2024-11-14T06:16:33,880 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T06:16:33,880 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T06:16:33,880 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:16:33,880 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:16:33,880 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:16:33,885 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-14T06:16:33,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T06:16:33,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-14T06:16:33,899 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731564993899"}]},"ts":"1731564993899"} 2024-11-14T06:16:33,902 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T06:16:33,902 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T06:16:33,904 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T06:16:33,910 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d7851abd86d644f97c54179c322b1c1e, UNASSIGN}] 2024-11-14T06:16:33,912 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d7851abd86d644f97c54179c322b1c1e, UNASSIGN 2024-11-14T06:16:33,914 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=d7851abd86d644f97c54179c322b1c1e, regionState=CLOSING, regionLocation=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:33,920 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d7851abd86d644f97c54179c322b1c1e, UNASSIGN because future has completed 2024-11-14T06:16:33,928 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T06:16:33,929 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945}] 2024-11-14T06:16:33,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-14T06:16:34,094 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(122): Close d7851abd86d644f97c54179c322b1c1e 2024-11-14T06:16:34,094 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T06:16:34,094 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1722): Closing d7851abd86d644f97c54179c322b1c1e, disabling compactions & flushes 2024-11-14T06:16:34,094 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:34,094 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:34,094 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. after waiting 0 ms 2024-11-14T06:16:34,094 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:34,095 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(2902): Flushing d7851abd86d644f97c54179c322b1c1e 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-14T06:16:34,095 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=A 2024-11-14T06:16:34,095 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:34,095 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=B 2024-11-14T06:16:34,095 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:34,095 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d7851abd86d644f97c54179c322b1c1e, store=C 2024-11-14T06:16:34,095 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:34,100 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/45810e1fc3f74fe6a0fcc00fe7e1f1f0 is 50, key is test_row_0/A:col10/1731564993007/Put/seqid=0 2024-11-14T06:16:34,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741945_1121 (size=12301) 2024-11-14T06:16:34,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-14T06:16:34,282 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T06:16:34,507 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=519 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/45810e1fc3f74fe6a0fcc00fe7e1f1f0 2024-11-14T06:16:34,515 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/07474583a90d46b49eadca60545128d2 is 50, key is test_row_0/B:col10/1731564993007/Put/seqid=0 2024-11-14T06:16:34,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-14T06:16:34,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741946_1122 (size=12301) 2024-11-14T06:16:34,520 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=519 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/07474583a90d46b49eadca60545128d2 2024-11-14T06:16:34,530 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/aa64b15e6ec74fc88c5dd85ddd261943 is 50, key is test_row_0/C:col10/1731564993007/Put/seqid=0 2024-11-14T06:16:34,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741947_1123 (size=12301) 2024-11-14T06:16:34,935 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=519 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/aa64b15e6ec74fc88c5dd85ddd261943 2024-11-14T06:16:34,941 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/A/45810e1fc3f74fe6a0fcc00fe7e1f1f0 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/45810e1fc3f74fe6a0fcc00fe7e1f1f0 2024-11-14T06:16:34,947 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/45810e1fc3f74fe6a0fcc00fe7e1f1f0, entries=150, sequenceid=519, filesize=12.0 K 2024-11-14T06:16:34,948 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/B/07474583a90d46b49eadca60545128d2 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/07474583a90d46b49eadca60545128d2 2024-11-14T06:16:34,954 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/07474583a90d46b49eadca60545128d2, entries=150, sequenceid=519, filesize=12.0 K 2024-11-14T06:16:34,955 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/.tmp/C/aa64b15e6ec74fc88c5dd85ddd261943 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/aa64b15e6ec74fc88c5dd85ddd261943 2024-11-14T06:16:34,963 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/aa64b15e6ec74fc88c5dd85ddd261943, entries=150, sequenceid=519, filesize=12.0 K 2024-11-14T06:16:34,965 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for d7851abd86d644f97c54179c322b1c1e in 870ms, sequenceid=519, compaction requested=true 2024-11-14T06:16:34,965 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/af35530d29884ab99f3e8671e990cf3e, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/3f3f93eafcd94ee9b649a4b323167c3c, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/95788cae5bb34c848cfa2742fc1a674f, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/0c834b1792724622a21cb2148c1bf1bf, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/16408aa6ee244b21bbc620db4f11ddd4, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/076c9070dc7b4233a87552e384aa22e9, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/11d7d3ab91a6490b9e6c598cb8fc713c, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/170d5fcbeed741cdb7fb3e71b8082d87, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/dac0f0fc524f47bbbce528f6d564924b, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/cedf4b535a064b74bff85cbe09ff2c56, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/8e6d8ad5a2f94c649051d403cab83cd4, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/bbf5047c48354d9d99f9cf697cd7335f, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/dbead1a3634346b29b1f95558d23fa76, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/915b87c2f5b74ddfabdc6b7218354440, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/27afff0584af472995f9addfb8b48bc4, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/e92b56aa17964df59d871ac4b0db286e, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/ea919ccc497d4edf89308816b93fffb9, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/e4dc9babe1184bef8cbe6a7498b437ac, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/52bc365df5a04c08b1ff224e72c120a1, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/ac049d36ac8743da968b08e3b7f247da, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/c9623358f1e845d187355d2f77d0816e, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/e538752ca4d04a529710b8f50772d642, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/eb6b265958ff48f18fb27cf7e284ca02, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/4b8f99c377544b619470832f84be5991, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/3c720d0ece964299bf31b9016dd94247, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/2165039abefd4e1f993da62a4f89bb44, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/3aedc79178344f0cb15dc520dbab081e, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/742db572e4a3425bb1cd2d97ab808016, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/d27b070d66654bae9c5539b3da3bbb18, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/d8b54246acc0450f9f707be0a92cf9d9, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/59b11410b705450daae5f3dd49736739, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/ab801cb7ada24b73ba7759e7c4f09f5b, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/1f2a1b788cc9483391c0b77ba500d125] to archive 2024-11-14T06:16:34,968 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T06:16:34,988 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/af35530d29884ab99f3e8671e990cf3e to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/af35530d29884ab99f3e8671e990cf3e 2024-11-14T06:16:34,991 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/3f3f93eafcd94ee9b649a4b323167c3c to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/3f3f93eafcd94ee9b649a4b323167c3c 2024-11-14T06:16:34,993 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/95788cae5bb34c848cfa2742fc1a674f to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/95788cae5bb34c848cfa2742fc1a674f 2024-11-14T06:16:34,994 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/0c834b1792724622a21cb2148c1bf1bf to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/0c834b1792724622a21cb2148c1bf1bf 2024-11-14T06:16:34,995 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/16408aa6ee244b21bbc620db4f11ddd4 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/16408aa6ee244b21bbc620db4f11ddd4 2024-11-14T06:16:34,997 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/076c9070dc7b4233a87552e384aa22e9 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/076c9070dc7b4233a87552e384aa22e9 2024-11-14T06:16:34,999 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/11d7d3ab91a6490b9e6c598cb8fc713c to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/11d7d3ab91a6490b9e6c598cb8fc713c 2024-11-14T06:16:35,000 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/170d5fcbeed741cdb7fb3e71b8082d87 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/170d5fcbeed741cdb7fb3e71b8082d87 2024-11-14T06:16:35,002 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/dac0f0fc524f47bbbce528f6d564924b to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/dac0f0fc524f47bbbce528f6d564924b 2024-11-14T06:16:35,004 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/cedf4b535a064b74bff85cbe09ff2c56 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/cedf4b535a064b74bff85cbe09ff2c56 2024-11-14T06:16:35,006 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/8e6d8ad5a2f94c649051d403cab83cd4 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/8e6d8ad5a2f94c649051d403cab83cd4 2024-11-14T06:16:35,008 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/bbf5047c48354d9d99f9cf697cd7335f to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/bbf5047c48354d9d99f9cf697cd7335f 2024-11-14T06:16:35,010 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/dbead1a3634346b29b1f95558d23fa76 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/dbead1a3634346b29b1f95558d23fa76 2024-11-14T06:16:35,013 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/915b87c2f5b74ddfabdc6b7218354440 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/915b87c2f5b74ddfabdc6b7218354440 2024-11-14T06:16:35,015 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/27afff0584af472995f9addfb8b48bc4 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/27afff0584af472995f9addfb8b48bc4 2024-11-14T06:16:35,017 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/e92b56aa17964df59d871ac4b0db286e to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/e92b56aa17964df59d871ac4b0db286e 2024-11-14T06:16:35,019 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/ea919ccc497d4edf89308816b93fffb9 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/ea919ccc497d4edf89308816b93fffb9 2024-11-14T06:16:35,021 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/e4dc9babe1184bef8cbe6a7498b437ac to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/e4dc9babe1184bef8cbe6a7498b437ac 2024-11-14T06:16:35,023 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/52bc365df5a04c08b1ff224e72c120a1 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/52bc365df5a04c08b1ff224e72c120a1 2024-11-14T06:16:35,025 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/ac049d36ac8743da968b08e3b7f247da to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/ac049d36ac8743da968b08e3b7f247da 2024-11-14T06:16:35,027 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/c9623358f1e845d187355d2f77d0816e to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/c9623358f1e845d187355d2f77d0816e 2024-11-14T06:16:35,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-14T06:16:35,028 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/e538752ca4d04a529710b8f50772d642 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/e538752ca4d04a529710b8f50772d642 2024-11-14T06:16:35,030 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/eb6b265958ff48f18fb27cf7e284ca02 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/eb6b265958ff48f18fb27cf7e284ca02 2024-11-14T06:16:35,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/4b8f99c377544b619470832f84be5991 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/4b8f99c377544b619470832f84be5991 2024-11-14T06:16:35,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/3c720d0ece964299bf31b9016dd94247 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/3c720d0ece964299bf31b9016dd94247 2024-11-14T06:16:35,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/2165039abefd4e1f993da62a4f89bb44 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/2165039abefd4e1f993da62a4f89bb44 2024-11-14T06:16:35,036 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/3aedc79178344f0cb15dc520dbab081e to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/3aedc79178344f0cb15dc520dbab081e 2024-11-14T06:16:35,038 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/742db572e4a3425bb1cd2d97ab808016 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/742db572e4a3425bb1cd2d97ab808016 2024-11-14T06:16:35,040 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/d27b070d66654bae9c5539b3da3bbb18 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/d27b070d66654bae9c5539b3da3bbb18 2024-11-14T06:16:35,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/d8b54246acc0450f9f707be0a92cf9d9 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/d8b54246acc0450f9f707be0a92cf9d9 2024-11-14T06:16:35,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/59b11410b705450daae5f3dd49736739 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/59b11410b705450daae5f3dd49736739 2024-11-14T06:16:35,044 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/ab801cb7ada24b73ba7759e7c4f09f5b to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/ab801cb7ada24b73ba7759e7c4f09f5b 2024-11-14T06:16:35,046 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/1f2a1b788cc9483391c0b77ba500d125 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/1f2a1b788cc9483391c0b77ba500d125 2024-11-14T06:16:35,069 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/1f563e6300eb4128b97d7b2e5212cc22, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/20c53261f4ef42a2b16bc9f1f6198b4b, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/a9081c74a881440c8ab00f9fcb0a7fff, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/65d479ab5cde4fafb53e84f46d8adb90, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/99fa4625cae249a2bb0dc0319d3e738a, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/01e3ddeefabe44b9b0bb779bf4ee3d4e, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/8f82a4635d8b40b59c53584415923c62, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/73ce2a6aa2ad4d519ba9fea5d618943d, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/1c5d2474929e48cbb5ccea57ebd46638, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/d14dd429dbb14487a243d5f21a0dab56, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/87d78efde7e949ea8a5a3dc793592d3c, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/0d40018fb1564f6e9bd47d0eeca33170, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/ebe2c26aa8e64e5daf006f8bbacc1fb2, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/153e629ab80d462a862a6647639dc963, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/d0dbf4cfdb434649a9e643ee7fc9d8f3, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/5193d475234e4b579aa92a33fad66a92, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/91f78e57640b42c59469054125e7bbb3, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/ad0695ccf950460cba14cfa80724d393, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/0dde91185dae4a5baccc13edb989cd2a, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/ac0d1c658cae4284a1ada552615673eb, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/b207d6b46cb14fe2a1ee9a61b4f8f714, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/eecf806f76dd4cab8d0f1300fe054e48, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/a9babefd97254158a25b1764ee9741fb, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/cbd1d52572624906b27eac2136e31b1c, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/aad6656df5da42168acb9eef03242cf8, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/9e0435bdfcd043e89bea854a2d7a4110, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/1788cd6470724dd599006d151f3487da, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/3242feb0dd8943e5bdc5e89efbf3bc6c, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/1715a6cca5b643f3be5f0473471a93f6, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/5d5c93fd45504cbbb3a00c96f78f3d4f, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/5803fae03a8f4e5ead99d48b7692b186, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/c4a05ae4a16a45a08536fbf1ac21b27f, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/93430d58e79749c9a5091c8805b0be83] to archive 2024-11-14T06:16:35,071 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T06:16:35,074 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/1f563e6300eb4128b97d7b2e5212cc22 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/1f563e6300eb4128b97d7b2e5212cc22 2024-11-14T06:16:35,076 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/20c53261f4ef42a2b16bc9f1f6198b4b to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/20c53261f4ef42a2b16bc9f1f6198b4b 2024-11-14T06:16:35,081 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/a9081c74a881440c8ab00f9fcb0a7fff to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/a9081c74a881440c8ab00f9fcb0a7fff 2024-11-14T06:16:35,083 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/65d479ab5cde4fafb53e84f46d8adb90 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/65d479ab5cde4fafb53e84f46d8adb90 2024-11-14T06:16:35,086 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/99fa4625cae249a2bb0dc0319d3e738a to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/99fa4625cae249a2bb0dc0319d3e738a 2024-11-14T06:16:35,097 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/01e3ddeefabe44b9b0bb779bf4ee3d4e to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/01e3ddeefabe44b9b0bb779bf4ee3d4e 2024-11-14T06:16:35,101 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/8f82a4635d8b40b59c53584415923c62 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/8f82a4635d8b40b59c53584415923c62 2024-11-14T06:16:35,105 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/73ce2a6aa2ad4d519ba9fea5d618943d to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/73ce2a6aa2ad4d519ba9fea5d618943d 2024-11-14T06:16:35,108 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/1c5d2474929e48cbb5ccea57ebd46638 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/1c5d2474929e48cbb5ccea57ebd46638 2024-11-14T06:16:35,113 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/d14dd429dbb14487a243d5f21a0dab56 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/d14dd429dbb14487a243d5f21a0dab56 2024-11-14T06:16:35,116 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/87d78efde7e949ea8a5a3dc793592d3c to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/87d78efde7e949ea8a5a3dc793592d3c 2024-11-14T06:16:35,128 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/0d40018fb1564f6e9bd47d0eeca33170 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/0d40018fb1564f6e9bd47d0eeca33170 2024-11-14T06:16:35,131 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/ebe2c26aa8e64e5daf006f8bbacc1fb2 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/ebe2c26aa8e64e5daf006f8bbacc1fb2 2024-11-14T06:16:35,133 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/153e629ab80d462a862a6647639dc963 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/153e629ab80d462a862a6647639dc963 2024-11-14T06:16:35,137 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/d0dbf4cfdb434649a9e643ee7fc9d8f3 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/d0dbf4cfdb434649a9e643ee7fc9d8f3 2024-11-14T06:16:35,140 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/5193d475234e4b579aa92a33fad66a92 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/5193d475234e4b579aa92a33fad66a92 2024-11-14T06:16:35,145 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/91f78e57640b42c59469054125e7bbb3 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/91f78e57640b42c59469054125e7bbb3 2024-11-14T06:16:35,147 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/ad0695ccf950460cba14cfa80724d393 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/ad0695ccf950460cba14cfa80724d393 2024-11-14T06:16:35,159 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/0dde91185dae4a5baccc13edb989cd2a to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/0dde91185dae4a5baccc13edb989cd2a 2024-11-14T06:16:35,173 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/ac0d1c658cae4284a1ada552615673eb to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/ac0d1c658cae4284a1ada552615673eb 2024-11-14T06:16:35,175 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/b207d6b46cb14fe2a1ee9a61b4f8f714 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/b207d6b46cb14fe2a1ee9a61b4f8f714 2024-11-14T06:16:35,177 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/eecf806f76dd4cab8d0f1300fe054e48 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/eecf806f76dd4cab8d0f1300fe054e48 2024-11-14T06:16:35,180 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/a9babefd97254158a25b1764ee9741fb to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/a9babefd97254158a25b1764ee9741fb 2024-11-14T06:16:35,182 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/cbd1d52572624906b27eac2136e31b1c to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/cbd1d52572624906b27eac2136e31b1c 2024-11-14T06:16:35,184 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/aad6656df5da42168acb9eef03242cf8 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/aad6656df5da42168acb9eef03242cf8 2024-11-14T06:16:35,192 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/9e0435bdfcd043e89bea854a2d7a4110 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/9e0435bdfcd043e89bea854a2d7a4110 2024-11-14T06:16:35,198 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/1788cd6470724dd599006d151f3487da to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/1788cd6470724dd599006d151f3487da 2024-11-14T06:16:35,200 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/3242feb0dd8943e5bdc5e89efbf3bc6c to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/3242feb0dd8943e5bdc5e89efbf3bc6c 2024-11-14T06:16:35,203 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/1715a6cca5b643f3be5f0473471a93f6 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/1715a6cca5b643f3be5f0473471a93f6 2024-11-14T06:16:35,205 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/5d5c93fd45504cbbb3a00c96f78f3d4f to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/5d5c93fd45504cbbb3a00c96f78f3d4f 2024-11-14T06:16:35,207 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/5803fae03a8f4e5ead99d48b7692b186 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/5803fae03a8f4e5ead99d48b7692b186 2024-11-14T06:16:35,209 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/c4a05ae4a16a45a08536fbf1ac21b27f to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/c4a05ae4a16a45a08536fbf1ac21b27f 2024-11-14T06:16:35,213 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/93430d58e79749c9a5091c8805b0be83 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/93430d58e79749c9a5091c8805b0be83 2024-11-14T06:16:35,217 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/0a933fe3e7754ed994c3e0ff2d4e1b33, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/16213f65774845a0b74d3d15ea6ae263, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/5a5b9008a5aa41f09b69efab83710373, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/a0ebc431ebb34176afa2c13ad4d214a7, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/f0d404a8188b43689d6223f3cf7db3ff, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/d71c253f64d14777b77f41b83ebceabc, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/cf9daf43a6984455b4a9bb3e71681e7a, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/1c9508b2673445e58e886f51e9712b78, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/f25b70a6e6224683b471f27faf7122f0, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/85e3a4dd2df7471dba782e9613e1cebb, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/bcaca51cb497417fa62e2168fa7ceea4, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/0ce19d0e90d54864a8b7a9848093b3a0, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/0162a3e74f184f49a26f74983c6ce4b6, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/e2edec5524844802a3fc00ed729eb767, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/616b2d8d3cc1447cadb67c961cc434eb, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/6cb882d79d8f4c5b84ee0f1667c79315, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/bb35d28de4fe4c8ab7a94a99bac56bd1, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/20628b4b0bd64a058f96c6d7e788a90d, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/86a4590700f64bf98230d4c73d7571c2, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/c6287bb5179b466f9e8a78fc689fce5d, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/9fc1f894f35e4167b36dccdcb330aef0, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/85306f34cdd44ac49b949333a326e121, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/3074e07bef3f4ba39489fae3dd75c843, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/9bbed59575a7411cb7c02d14b1c453ac, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/bc9e694409584947aaea92a3c1e01e8d, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/c3fa25b1cea94818bfe48c384f3051fc, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/f82a1de4495548318341f0f8da0c9673, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/4cf065cc93284835a08c11ff91ab262f, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/e15e4ef541da4d3580e0400b8215a519, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/dde1275ca2ac4f398128daddf3a25849, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/f5dab5c14e2248cc8f4eeb4a407072dd, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/575f6a6bbb294460859aa8a926982506, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/36b9566d40e145b4889fec1f5826fc37] to archive 2024-11-14T06:16:35,218 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T06:16:35,220 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/0a933fe3e7754ed994c3e0ff2d4e1b33 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/0a933fe3e7754ed994c3e0ff2d4e1b33 2024-11-14T06:16:35,221 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/16213f65774845a0b74d3d15ea6ae263 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/16213f65774845a0b74d3d15ea6ae263 2024-11-14T06:16:35,223 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/5a5b9008a5aa41f09b69efab83710373 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/5a5b9008a5aa41f09b69efab83710373 2024-11-14T06:16:35,224 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/a0ebc431ebb34176afa2c13ad4d214a7 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/a0ebc431ebb34176afa2c13ad4d214a7 2024-11-14T06:16:35,225 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/f0d404a8188b43689d6223f3cf7db3ff to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/f0d404a8188b43689d6223f3cf7db3ff 2024-11-14T06:16:35,227 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/d71c253f64d14777b77f41b83ebceabc to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/d71c253f64d14777b77f41b83ebceabc 2024-11-14T06:16:35,229 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/cf9daf43a6984455b4a9bb3e71681e7a to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/cf9daf43a6984455b4a9bb3e71681e7a 2024-11-14T06:16:35,236 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/1c9508b2673445e58e886f51e9712b78 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/1c9508b2673445e58e886f51e9712b78 2024-11-14T06:16:35,238 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/f25b70a6e6224683b471f27faf7122f0 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/f25b70a6e6224683b471f27faf7122f0 2024-11-14T06:16:35,240 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/85e3a4dd2df7471dba782e9613e1cebb to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/85e3a4dd2df7471dba782e9613e1cebb 2024-11-14T06:16:35,243 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/bcaca51cb497417fa62e2168fa7ceea4 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/bcaca51cb497417fa62e2168fa7ceea4 2024-11-14T06:16:35,245 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/0ce19d0e90d54864a8b7a9848093b3a0 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/0ce19d0e90d54864a8b7a9848093b3a0 2024-11-14T06:16:35,248 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/0162a3e74f184f49a26f74983c6ce4b6 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/0162a3e74f184f49a26f74983c6ce4b6 2024-11-14T06:16:35,250 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/e2edec5524844802a3fc00ed729eb767 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/e2edec5524844802a3fc00ed729eb767 2024-11-14T06:16:35,252 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/616b2d8d3cc1447cadb67c961cc434eb to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/616b2d8d3cc1447cadb67c961cc434eb 2024-11-14T06:16:35,254 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/6cb882d79d8f4c5b84ee0f1667c79315 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/6cb882d79d8f4c5b84ee0f1667c79315 2024-11-14T06:16:35,256 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/bb35d28de4fe4c8ab7a94a99bac56bd1 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/bb35d28de4fe4c8ab7a94a99bac56bd1 2024-11-14T06:16:35,258 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/20628b4b0bd64a058f96c6d7e788a90d to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/20628b4b0bd64a058f96c6d7e788a90d 2024-11-14T06:16:35,260 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/86a4590700f64bf98230d4c73d7571c2 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/86a4590700f64bf98230d4c73d7571c2 2024-11-14T06:16:35,262 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/c6287bb5179b466f9e8a78fc689fce5d to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/c6287bb5179b466f9e8a78fc689fce5d 2024-11-14T06:16:35,264 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/9fc1f894f35e4167b36dccdcb330aef0 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/9fc1f894f35e4167b36dccdcb330aef0 2024-11-14T06:16:35,266 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/85306f34cdd44ac49b949333a326e121 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/85306f34cdd44ac49b949333a326e121 2024-11-14T06:16:35,268 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/3074e07bef3f4ba39489fae3dd75c843 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/3074e07bef3f4ba39489fae3dd75c843 2024-11-14T06:16:35,271 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/9bbed59575a7411cb7c02d14b1c453ac to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/9bbed59575a7411cb7c02d14b1c453ac 2024-11-14T06:16:35,277 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/bc9e694409584947aaea92a3c1e01e8d to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/bc9e694409584947aaea92a3c1e01e8d 2024-11-14T06:16:35,289 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/c3fa25b1cea94818bfe48c384f3051fc to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/c3fa25b1cea94818bfe48c384f3051fc 2024-11-14T06:16:35,291 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/f82a1de4495548318341f0f8da0c9673 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/f82a1de4495548318341f0f8da0c9673 2024-11-14T06:16:35,299 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/4cf065cc93284835a08c11ff91ab262f to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/4cf065cc93284835a08c11ff91ab262f 2024-11-14T06:16:35,306 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/e15e4ef541da4d3580e0400b8215a519 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/e15e4ef541da4d3580e0400b8215a519 2024-11-14T06:16:35,310 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/dde1275ca2ac4f398128daddf3a25849 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/dde1275ca2ac4f398128daddf3a25849 2024-11-14T06:16:35,312 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/f5dab5c14e2248cc8f4eeb4a407072dd to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/f5dab5c14e2248cc8f4eeb4a407072dd 2024-11-14T06:16:35,314 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/575f6a6bbb294460859aa8a926982506 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/575f6a6bbb294460859aa8a926982506 2024-11-14T06:16:35,316 DEBUG [StoreCloser-TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/36b9566d40e145b4889fec1f5826fc37 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/36b9566d40e145b4889fec1f5826fc37 2024-11-14T06:16:35,342 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/recovered.edits/522.seqid, newMaxSeqId=522, maxSeqId=1 2024-11-14T06:16:35,345 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. 2024-11-14T06:16:35,345 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1676): Region close journal for d7851abd86d644f97c54179c322b1c1e: Waiting for close lock at 1731564994094Running coprocessor pre-close hooks at 1731564994094Disabling compacts and flushes for region at 1731564994094Disabling writes for close at 1731564994094Obtaining lock to block concurrent updates at 1731564994095 (+1 ms)Preparing flush snapshotting stores in d7851abd86d644f97c54179c322b1c1e at 1731564994095Finished memstore snapshotting TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731564994095Flushing stores of TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e. at 1731564994096 (+1 ms)Flushing d7851abd86d644f97c54179c322b1c1e/A: creating writer at 1731564994096Flushing d7851abd86d644f97c54179c322b1c1e/A: appending metadata at 1731564994099 (+3 ms)Flushing d7851abd86d644f97c54179c322b1c1e/A: closing flushed file at 1731564994099Flushing d7851abd86d644f97c54179c322b1c1e/B: creating writer at 1731564994512 (+413 ms)Flushing d7851abd86d644f97c54179c322b1c1e/B: appending metadata at 1731564994515 (+3 ms)Flushing d7851abd86d644f97c54179c322b1c1e/B: closing flushed file at 1731564994515Flushing d7851abd86d644f97c54179c322b1c1e/C: creating writer at 1731564994526 (+11 ms)Flushing d7851abd86d644f97c54179c322b1c1e/C: appending metadata at 1731564994529 (+3 ms)Flushing d7851abd86d644f97c54179c322b1c1e/C: closing flushed file at 1731564994529Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6e48afcd: reopening flushed file at 1731564994940 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@135a1185: reopening flushed file at 1731564994947 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@76ac0f0c: reopening flushed file at 1731564994954 (+7 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for d7851abd86d644f97c54179c322b1c1e in 870ms, sequenceid=519, compaction requested=true at 1731564994965 (+11 ms)Writing region close event to WAL at 1731564995331 (+366 ms)Running coprocessor post-close hooks at 1731564995345 (+14 ms)Closed at 1731564995345 2024-11-14T06:16:35,349 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(157): Closed d7851abd86d644f97c54179c322b1c1e 2024-11-14T06:16:35,350 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=d7851abd86d644f97c54179c322b1c1e, regionState=CLOSED 2024-11-14T06:16:35,357 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 because future has completed 2024-11-14T06:16:35,368 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-14T06:16:35,369 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseRegionProcedure d7851abd86d644f97c54179c322b1c1e, server=ef93ef3a83c5,42283,1731564965945 in 1.4340 sec 2024-11-14T06:16:35,371 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-14T06:16:35,371 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d7851abd86d644f97c54179c322b1c1e, UNASSIGN in 1.4590 sec 2024-11-14T06:16:35,374 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-14T06:16:35,374 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4680 sec 2024-11-14T06:16:35,378 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731564995378"}]},"ts":"1731564995378"} 2024-11-14T06:16:35,382 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T06:16:35,382 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T06:16:35,388 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4960 sec 2024-11-14T06:16:35,538 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-14T06:16:36,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-14T06:16:36,038 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:16:36,041 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-14T06:16:36,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T06:16:36,048 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T06:16:36,050 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=29, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T06:16:36,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-14T06:16:36,056 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e 2024-11-14T06:16:36,061 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A, FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B, FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C, FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/recovered.edits] 2024-11-14T06:16:36,064 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/09f394e03b72477c984387486680f0dc to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/09f394e03b72477c984387486680f0dc 2024-11-14T06:16:36,066 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/45810e1fc3f74fe6a0fcc00fe7e1f1f0 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/45810e1fc3f74fe6a0fcc00fe7e1f1f0 2024-11-14T06:16:36,067 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/748a896e4404465fb3541699a6c4e0b9 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/748a896e4404465fb3541699a6c4e0b9 2024-11-14T06:16:36,068 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/87511dfe2fe3404da250f090eddb3d1d to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/A/87511dfe2fe3404da250f090eddb3d1d 2024-11-14T06:16:36,071 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/07474583a90d46b49eadca60545128d2 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/07474583a90d46b49eadca60545128d2 2024-11-14T06:16:36,072 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/98940bed0fd443dca349244266a183db to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/98940bed0fd443dca349244266a183db 2024-11-14T06:16:36,074 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/a57658a5458645b8a17f839ea011b1e1 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/a57658a5458645b8a17f839ea011b1e1 2024-11-14T06:16:36,076 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/f3aee24b80754aae8998a26e5e114b31 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/B/f3aee24b80754aae8998a26e5e114b31 2024-11-14T06:16:36,079 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/42c6ada6cfe048dc91560b39f5794930 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/42c6ada6cfe048dc91560b39f5794930 2024-11-14T06:16:36,081 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/637ad1cdf64248ea872d9d33614071bc to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/637ad1cdf64248ea872d9d33614071bc 2024-11-14T06:16:36,082 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/9d15e92ba0da41bebb83fce36483d4ab to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/9d15e92ba0da41bebb83fce36483d4ab 2024-11-14T06:16:36,084 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/aa64b15e6ec74fc88c5dd85ddd261943 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/C/aa64b15e6ec74fc88c5dd85ddd261943 2024-11-14T06:16:36,087 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/recovered.edits/522.seqid to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e/recovered.edits/522.seqid 2024-11-14T06:16:36,088 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d7851abd86d644f97c54179c322b1c1e 2024-11-14T06:16:36,088 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T06:16:36,092 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=29, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T06:16:36,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-14T06:16:36,103 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T06:16:36,132 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T06:16:36,134 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=29, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T06:16:36,134 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T06:16:36,134 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731564996134"}]},"ts":"9223372036854775807"} 2024-11-14T06:16:36,138 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T06:16:36,138 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => d7851abd86d644f97c54179c322b1c1e, NAME => 'TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T06:16:36,139 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T06:16:36,139 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731564996139"}]},"ts":"9223372036854775807"} 2024-11-14T06:16:36,142 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T06:16:36,143 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=29, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T06:16:36,144 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 101 msec 2024-11-14T06:16:36,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-14T06:16:36,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T06:16:36,158 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:16:36,174 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=249 (was 211) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-20636308_22 at /127.0.0.1:52842 [Waiting for operation #293] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_37528205_22 at /127.0.0.1:46640 [Waiting for operation #130] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_37528205_22 at /127.0.0.1:43780 [Waiting for operation #271] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;ef93ef3a83c5:42283-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=461 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=668 (was 231) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1899 (was 3518) 2024-11-14T06:16:36,184 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=249, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=668, ProcessCount=11, AvailableMemoryMB=1898 2024-11-14T06:16:36,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T06:16:36,185 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T06:16:36,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T06:16:36,188 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T06:16:36,188 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:36,188 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 30 2024-11-14T06:16:36,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-14T06:16:36,189 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T06:16:36,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741948_1124 (size=1061) 2024-11-14T06:16:36,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-14T06:16:36,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-14T06:16:36,610 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02 2024-11-14T06:16:36,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741949_1125 (size=53) 2024-11-14T06:16:36,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-14T06:16:37,019 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T06:16:37,019 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 715443fb55c29fd4a00fd2e9afbe3938, disabling compactions & flushes 2024-11-14T06:16:37,019 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:37,019 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:37,019 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. after waiting 0 ms 2024-11-14T06:16:37,019 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:37,019 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:37,019 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 715443fb55c29fd4a00fd2e9afbe3938: Waiting for close lock at 1731564997019Disabling compacts and flushes for region at 1731564997019Disabling writes for close at 1731564997019Writing region close event to WAL at 1731564997019Closed at 1731564997019 2024-11-14T06:16:37,021 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T06:16:37,021 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731564997021"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731564997021"}]},"ts":"1731564997021"} 2024-11-14T06:16:37,023 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T06:16:37,024 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T06:16:37,024 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731564997024"}]},"ts":"1731564997024"} 2024-11-14T06:16:37,027 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T06:16:37,027 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=715443fb55c29fd4a00fd2e9afbe3938, ASSIGN}] 2024-11-14T06:16:37,028 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=715443fb55c29fd4a00fd2e9afbe3938, ASSIGN 2024-11-14T06:16:37,029 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=715443fb55c29fd4a00fd2e9afbe3938, ASSIGN; state=OFFLINE, location=ef93ef3a83c5,42283,1731564965945; forceNewPlan=false, retain=false 2024-11-14T06:16:37,180 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=715443fb55c29fd4a00fd2e9afbe3938, regionState=OPENING, regionLocation=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:37,182 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=715443fb55c29fd4a00fd2e9afbe3938, ASSIGN because future has completed 2024-11-14T06:16:37,182 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945}] 2024-11-14T06:16:37,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-14T06:16:37,338 INFO [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:37,338 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} 2024-11-14T06:16:37,339 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:37,339 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T06:16:37,339 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:37,339 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:37,340 INFO [StoreOpener-715443fb55c29fd4a00fd2e9afbe3938-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:37,341 INFO [StoreOpener-715443fb55c29fd4a00fd2e9afbe3938-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T06:16:37,342 INFO [StoreOpener-715443fb55c29fd4a00fd2e9afbe3938-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 715443fb55c29fd4a00fd2e9afbe3938 columnFamilyName A 2024-11-14T06:16:37,343 DEBUG [StoreOpener-715443fb55c29fd4a00fd2e9afbe3938-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:37,343 INFO [StoreOpener-715443fb55c29fd4a00fd2e9afbe3938-1 {}] regionserver.HStore(327): Store=715443fb55c29fd4a00fd2e9afbe3938/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T06:16:37,343 INFO [StoreOpener-715443fb55c29fd4a00fd2e9afbe3938-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:37,344 INFO [StoreOpener-715443fb55c29fd4a00fd2e9afbe3938-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T06:16:37,345 INFO [StoreOpener-715443fb55c29fd4a00fd2e9afbe3938-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 715443fb55c29fd4a00fd2e9afbe3938 columnFamilyName B 2024-11-14T06:16:37,345 DEBUG [StoreOpener-715443fb55c29fd4a00fd2e9afbe3938-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:37,345 INFO [StoreOpener-715443fb55c29fd4a00fd2e9afbe3938-1 {}] regionserver.HStore(327): Store=715443fb55c29fd4a00fd2e9afbe3938/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T06:16:37,346 INFO [StoreOpener-715443fb55c29fd4a00fd2e9afbe3938-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:37,347 INFO [StoreOpener-715443fb55c29fd4a00fd2e9afbe3938-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T06:16:37,347 INFO [StoreOpener-715443fb55c29fd4a00fd2e9afbe3938-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 715443fb55c29fd4a00fd2e9afbe3938 columnFamilyName C 2024-11-14T06:16:37,347 DEBUG [StoreOpener-715443fb55c29fd4a00fd2e9afbe3938-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:37,347 INFO [StoreOpener-715443fb55c29fd4a00fd2e9afbe3938-1 {}] regionserver.HStore(327): Store=715443fb55c29fd4a00fd2e9afbe3938/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T06:16:37,348 INFO [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:37,348 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:37,348 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:37,349 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:37,350 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:37,350 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:37,350 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T06:16:37,351 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:37,353 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T06:16:37,354 INFO [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 715443fb55c29fd4a00fd2e9afbe3938; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66304520, jitterRate=-0.0119856595993042}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T06:16:37,354 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:37,355 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 715443fb55c29fd4a00fd2e9afbe3938: Running coprocessor pre-open hook at 1731564997339Writing region info on filesystem at 1731564997339Initializing all the Stores at 1731564997340 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731564997340Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731564997340Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731564997340Cleaning up temporary data from old regions at 1731564997350 (+10 ms)Running coprocessor post-open hooks at 1731564997354 (+4 ms)Region opened successfully at 1731564997355 (+1 ms) 2024-11-14T06:16:37,356 INFO [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., pid=32, masterSystemTime=1731564997334 2024-11-14T06:16:37,358 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:37,358 INFO [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:37,359 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=715443fb55c29fd4a00fd2e9afbe3938, regionState=OPEN, openSeqNum=2, regionLocation=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:37,361 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 because future has completed 2024-11-14T06:16:37,364 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-14T06:16:37,364 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure 715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 in 180 msec 2024-11-14T06:16:37,367 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-14T06:16:37,367 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=715443fb55c29fd4a00fd2e9afbe3938, ASSIGN in 337 msec 2024-11-14T06:16:37,368 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T06:16:37,369 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731564997368"}]},"ts":"1731564997368"} 2024-11-14T06:16:37,371 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T06:16:37,372 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T06:16:37,374 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1860 sec 2024-11-14T06:16:38,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-14T06:16:38,338 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:16:38,339 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4aee7018, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:16:38,339 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:16:38,340 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:16:38,341 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:16:38,342 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:16:38,342 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:16:38,342 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17e2a619, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:16:38,342 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:16:38,342 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:16:38,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:16:38,344 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44142, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:16:38,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@641dcc92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:16:38,345 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:16:38,347 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:16:38,347 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:16:38,349 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43414, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:16:38,351 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T06:16:38,352 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is ef93ef3a83c5,36045,1731564965170 2024-11-14T06:16:38,352 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@296fe578 2024-11-14T06:16:38,352 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T06:16:38,353 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44144, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T06:16:38,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T06:16:38,359 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T06:16:38,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-14T06:16:38,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741950_1126 (size=1097) 2024-11-14T06:16:38,797 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-14T06:16:38,797 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-14T06:16:38,802 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T06:16:38,811 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=715443fb55c29fd4a00fd2e9afbe3938, REOPEN/MOVE}] 2024-11-14T06:16:38,812 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=715443fb55c29fd4a00fd2e9afbe3938, REOPEN/MOVE 2024-11-14T06:16:38,813 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=715443fb55c29fd4a00fd2e9afbe3938, regionState=CLOSING, regionLocation=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:38,815 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=715443fb55c29fd4a00fd2e9afbe3938, REOPEN/MOVE because future has completed 2024-11-14T06:16:38,816 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T06:16:38,816 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945}] 2024-11-14T06:16:38,969 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(122): Close 715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:38,970 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T06:16:38,970 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1722): Closing 715443fb55c29fd4a00fd2e9afbe3938, disabling compactions & flushes 2024-11-14T06:16:38,970 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:38,970 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:38,970 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. after waiting 0 ms 2024-11-14T06:16:38,970 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:38,977 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T06:16:38,978 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:38,978 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1676): Region close journal for 715443fb55c29fd4a00fd2e9afbe3938: Waiting for close lock at 1731564998970Running coprocessor pre-close hooks at 1731564998970Disabling compacts and flushes for region at 1731564998970Disabling writes for close at 1731564998970Writing region close event to WAL at 1731564998970Running coprocessor post-close hooks at 1731564998978 (+8 ms)Closed at 1731564998978 2024-11-14T06:16:38,978 WARN [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionServer(3299): Not adding moved region record: 715443fb55c29fd4a00fd2e9afbe3938 to self. 2024-11-14T06:16:38,982 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(157): Closed 715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:38,987 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=715443fb55c29fd4a00fd2e9afbe3938, regionState=CLOSED 2024-11-14T06:16:38,991 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 because future has completed 2024-11-14T06:16:39,014 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-14T06:16:39,015 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 in 195 msec 2024-11-14T06:16:39,015 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=715443fb55c29fd4a00fd2e9afbe3938, REOPEN/MOVE; state=CLOSED, location=ef93ef3a83c5,42283,1731564965945; forceNewPlan=false, retain=true 2024-11-14T06:16:39,166 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=715443fb55c29fd4a00fd2e9afbe3938, regionState=OPENING, regionLocation=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:39,168 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=715443fb55c29fd4a00fd2e9afbe3938, REOPEN/MOVE because future has completed 2024-11-14T06:16:39,169 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945}] 2024-11-14T06:16:39,324 INFO [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:39,324 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} 2024-11-14T06:16:39,325 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:39,325 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T06:16:39,325 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:39,325 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:39,329 INFO [StoreOpener-715443fb55c29fd4a00fd2e9afbe3938-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:39,330 INFO [StoreOpener-715443fb55c29fd4a00fd2e9afbe3938-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T06:16:39,334 INFO [StoreOpener-715443fb55c29fd4a00fd2e9afbe3938-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 715443fb55c29fd4a00fd2e9afbe3938 columnFamilyName A 2024-11-14T06:16:39,336 DEBUG [StoreOpener-715443fb55c29fd4a00fd2e9afbe3938-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:39,337 INFO [StoreOpener-715443fb55c29fd4a00fd2e9afbe3938-1 {}] regionserver.HStore(327): Store=715443fb55c29fd4a00fd2e9afbe3938/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T06:16:39,338 INFO [StoreOpener-715443fb55c29fd4a00fd2e9afbe3938-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:39,338 INFO [StoreOpener-715443fb55c29fd4a00fd2e9afbe3938-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T06:16:39,339 INFO [StoreOpener-715443fb55c29fd4a00fd2e9afbe3938-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 715443fb55c29fd4a00fd2e9afbe3938 columnFamilyName B 2024-11-14T06:16:39,339 DEBUG [StoreOpener-715443fb55c29fd4a00fd2e9afbe3938-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:39,339 INFO [StoreOpener-715443fb55c29fd4a00fd2e9afbe3938-1 {}] regionserver.HStore(327): Store=715443fb55c29fd4a00fd2e9afbe3938/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T06:16:39,340 INFO [StoreOpener-715443fb55c29fd4a00fd2e9afbe3938-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:39,341 INFO [StoreOpener-715443fb55c29fd4a00fd2e9afbe3938-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T06:16:39,341 INFO [StoreOpener-715443fb55c29fd4a00fd2e9afbe3938-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 715443fb55c29fd4a00fd2e9afbe3938 columnFamilyName C 2024-11-14T06:16:39,341 DEBUG [StoreOpener-715443fb55c29fd4a00fd2e9afbe3938-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:39,342 INFO [StoreOpener-715443fb55c29fd4a00fd2e9afbe3938-1 {}] regionserver.HStore(327): Store=715443fb55c29fd4a00fd2e9afbe3938/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T06:16:39,342 INFO [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:39,342 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:39,342 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:39,344 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:39,345 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:39,345 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:39,345 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T06:16:39,346 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:39,347 INFO [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 715443fb55c29fd4a00fd2e9afbe3938; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70727071, jitterRate=0.05391548573970795}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T06:16:39,347 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:39,348 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 715443fb55c29fd4a00fd2e9afbe3938: Running coprocessor pre-open hook at 1731564999325Writing region info on filesystem at 1731564999325Initializing all the Stores at 1731564999326 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731564999326Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731564999326Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731564999326Cleaning up temporary data from old regions at 1731564999345 (+19 ms)Running coprocessor post-open hooks at 1731564999347 (+2 ms)Region opened successfully at 1731564999348 (+1 ms) 2024-11-14T06:16:39,349 INFO [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., pid=37, masterSystemTime=1731564999321 2024-11-14T06:16:39,351 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:39,351 INFO [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:39,352 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=715443fb55c29fd4a00fd2e9afbe3938, regionState=OPEN, openSeqNum=5, regionLocation=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:39,353 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 because future has completed 2024-11-14T06:16:39,356 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=35 2024-11-14T06:16:39,356 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure 715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 in 185 msec 2024-11-14T06:16:39,358 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-14T06:16:39,358 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=715443fb55c29fd4a00fd2e9afbe3938, REOPEN/MOVE in 545 msec 2024-11-14T06:16:39,361 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-14T06:16:39,361 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 558 msec 2024-11-14T06:16:39,365 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 1.0010 sec 2024-11-14T06:16:39,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-14T06:16:39,367 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:16:39,368 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fcb470e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:16:39,368 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:16:39,368 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:16:39,369 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:16:39,369 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:16:39,369 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:16:39,369 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@363fb04a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:16:39,369 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:16:39,369 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:16:39,370 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:16:39,371 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44162, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:16:39,371 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f5c30e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:16:39,372 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@779c6619, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:16:39,372 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:16:39,372 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:16:39,374 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:16:39,374 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:16:39,374 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:16:39,374 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d54e90b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:16:39,374 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:16:39,375 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:16:39,375 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:16:39,377 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44168, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:16:39,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ae156c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:16:39,380 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fe41499, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:16:39,380 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:16:39,380 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:16:39,381 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:16:39,381 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:16:39,382 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:16:39,382 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13bd06f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:16:39,382 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:16:39,382 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:16:39,383 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:16:39,384 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44172, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:16:39,386 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ff0502d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:16:39,388 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3921eb8f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:16:39,388 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:16:39,388 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:16:39,389 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:16:39,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:16:39,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:16:39,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1db814a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:16:39,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:16:39,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:16:39,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:16:39,391 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44184, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:16:39,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3585e846, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:16:39,395 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3621b0a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:16:39,395 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:16:39,395 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:16:39,396 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:16:39,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:16:39,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:16:39,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f638138, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:16:39,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:16:39,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:16:39,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:16:39,399 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44202, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:16:39,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@547ef88a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:16:39,401 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c57f23b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:16:39,401 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:16:39,402 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:16:39,402 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:16:39,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:16:39,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:16:39,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@724f60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:16:39,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:16:39,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:16:39,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:16:39,404 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44226, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:16:39,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2427f089, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:16:39,408 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2023ab21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:16:39,408 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:16:39,409 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:16:39,409 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:16:39,410 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:16:39,410 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:16:39,410 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@764a5610, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:16:39,410 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:16:39,410 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:16:39,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:16:39,411 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44248, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:16:39,412 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b0905f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:16:39,413 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@483c7a4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:16:39,413 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:16:39,413 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:16:39,414 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:16:39,415 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:16:39,415 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:16:39,415 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21c3e521, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:16:39,415 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:16:39,415 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:16:39,416 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:16:39,416 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44266, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:16:39,419 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dc3611c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:16:39,420 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6986f6f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:16:39,420 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:16:39,421 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:16:39,422 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:16:39,422 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:16:39,422 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:16:39,422 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c765914, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:16:39,422 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:16:39,423 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:16:39,423 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:16:39,425 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44284, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:16:39,426 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad43a4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:16:39,433 DEBUG [Thread-646 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:16:39,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:16:39,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:16:39,436 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43430, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:16:39,437 DEBUG [Thread-641 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:16:39,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:16:39,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:16:39,439 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5] 2024-11-14T06:16:39,439 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43444, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:16:39,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5] 2024-11-14T06:16:39,444 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T06:16:39,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-14T06:16:39,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-14T06:16:39,447 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T06:16:39,447 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T06:16:39,448 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T06:16:39,450 DEBUG [Thread-635 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:16:39,450 DEBUG [Thread-650 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:16:39,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:16:39,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:16:39,452 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:16:39,452 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:16:39,452 DEBUG [Thread-644 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:16:39,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:16:39,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:16:39,454 DEBUG [Thread-633 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:16:39,455 DEBUG [Thread-633 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:16:39,456 DEBUG [Thread-633 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:16:39,457 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43446, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:16:39,457 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43466, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:16:39,457 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43450, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:16:39,457 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43470, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:16:39,459 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5] 2024-11-14T06:16:39,461 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5] 2024-11-14T06:16:39,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5] 2024-11-14T06:16:39,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5] 2024-11-14T06:16:39,469 DEBUG [Thread-639 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:16:39,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:16:39,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:16:39,471 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43476, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:16:39,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5] 2024-11-14T06:16:39,476 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 715443fb55c29fd4a00fd2e9afbe3938 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T06:16:39,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on 715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:39,481 DEBUG [Thread-637 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:16:39,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:16:39,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:16:39,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 715443fb55c29fd4a00fd2e9afbe3938, store=A 2024-11-14T06:16:39,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:39,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 715443fb55c29fd4a00fd2e9afbe3938, store=B 2024-11-14T06:16:39,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:39,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 715443fb55c29fd4a00fd2e9afbe3938, store=C 2024-11-14T06:16:39,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:39,486 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43484, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:16:39,488 DEBUG [Thread-648 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:16:39,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5] 2024-11-14T06:16:39,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:16:39,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:16:39,492 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43490, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:16:39,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5] 2024-11-14T06:16:39,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114c355ba56b38e45568166f824d467f428_715443fb55c29fd4a00fd2e9afbe3938 is 50, key is test_row_0/A:col10/1731564999463/Put/seqid=0 2024-11-14T06:16:39,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-14T06:16:39,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741951_1127 (size=17034) 2024-11-14T06:16:39,605 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:39,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-14T06:16:39,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:39,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. as already flushing 2024-11-14T06:16:39,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:39,607 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:39,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:39,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:39,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:39,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43470 deadline: 1731565059597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:39,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:39,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:39,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:39,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:39,618 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731564999594, max timestamp=1731564999604Num uniques 859; 2024-11-14T06:16:39,619 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T06:16:39,619 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731564999594, max timestamp=1731564999604Num uniques 859; 2024-11-14T06:16:39,619 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T06:16:39,615 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731564999594, max timestamp=1731564999604Num uniques 859; 2024-11-14T06:16:39,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43484 deadline: 1731565059602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:39,619 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T06:16:39,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:39,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:39,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:39,634 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114c355ba56b38e45568166f824d467f428_715443fb55c29fd4a00fd2e9afbe3938 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c355ba56b38e45568166f824d467f428_715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:39,645 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/a1186be964b14f66a3b84dfe4966754c, store: [table=TestAcidGuarantees family=A region=715443fb55c29fd4a00fd2e9afbe3938] 2024-11-14T06:16:39,652 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/a1186be964b14f66a3b84dfe4966754c is 175, key is test_row_0/A:col10/1731564999463/Put/seqid=0 2024-11-14T06:16:39,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:39,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43446 deadline: 1731565059670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:39,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:39,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:39,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:39,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:39,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43476 deadline: 1731565059671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:39,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:39,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:39,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:39,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:39,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43444 deadline: 1731565059678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:39,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:39,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:39,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:39,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741952_1128 (size=48139) 2024-11-14T06:16:39,703 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/a1186be964b14f66a3b84dfe4966754c 2024-11-14T06:16:39,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:39,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43470 deadline: 1731565059728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:39,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:39,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:39,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:39,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:39,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43484 deadline: 1731565059728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:39,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:39,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:39,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:39,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-14T06:16:39,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:39,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. as already flushing 2024-11-14T06:16:39,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:39,765 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:39,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:39,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:39,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-14T06:16:39,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/9af36e19d06342168b217f9b8d0bcad4 is 50, key is test_row_0/B:col10/1731564999463/Put/seqid=0 2024-11-14T06:16:39,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741953_1129 (size=12001) 2024-11-14T06:16:39,780 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/9af36e19d06342168b217f9b8d0bcad4 2024-11-14T06:16:39,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:39,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43446 deadline: 1731565059788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:39,797 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:39,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:39,797 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:39,797 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:39,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43476 deadline: 1731565059788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:39,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:39,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:39,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:39,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:39,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43444 deadline: 1731565059797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:39,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:39,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:39,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:39,834 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/612597fe630b42ce89c04d714f8754ff is 50, key is test_row_0/C:col10/1731564999463/Put/seqid=0 2024-11-14T06:16:39,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741954_1130 (size=12001) 2024-11-14T06:16:39,885 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/612597fe630b42ce89c04d714f8754ff 2024-11-14T06:16:39,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/a1186be964b14f66a3b84dfe4966754c as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/a1186be964b14f66a3b84dfe4966754c 2024-11-14T06:16:39,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-14T06:16:39,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:39,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. as already flushing 2024-11-14T06:16:39,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:39,920 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:39,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:39,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:39,935 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/a1186be964b14f66a3b84dfe4966754c, entries=250, sequenceid=15, filesize=47.0 K 2024-11-14T06:16:39,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/9af36e19d06342168b217f9b8d0bcad4 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/9af36e19d06342168b217f9b8d0bcad4 2024-11-14T06:16:39,956 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/9af36e19d06342168b217f9b8d0bcad4, entries=150, sequenceid=15, filesize=11.7 K 2024-11-14T06:16:39,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/612597fe630b42ce89c04d714f8754ff as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/612597fe630b42ce89c04d714f8754ff 2024-11-14T06:16:39,966 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/612597fe630b42ce89c04d714f8754ff, entries=150, sequenceid=15, filesize=11.7 K 2024-11-14T06:16:39,967 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 715443fb55c29fd4a00fd2e9afbe3938 in 491ms, sequenceid=15, compaction requested=false 2024-11-14T06:16:39,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 715443fb55c29fd4a00fd2e9afbe3938: 2024-11-14T06:16:39,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on 715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:39,988 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 715443fb55c29fd4a00fd2e9afbe3938 3/3 column families, dataSize=134.18 KB heapSize=384.97 KB 2024-11-14T06:16:39,994 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731564999662, max timestamp=1731564999984Num uniques 1018; 2024-11-14T06:16:39,994 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-14T06:16:39,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 715443fb55c29fd4a00fd2e9afbe3938, store=A 2024-11-14T06:16:39,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:16:39,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 715443fb55c29fd4a00fd2e9afbe3938, store=B 2024-11-14T06:16:39,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-14T06:16:39,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 715443fb55c29fd4a00fd2e9afbe3938, store=C 2024-11-14T06:16:39,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-14T06:16:40,037 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411141433d3e589634021b628c96595e9fc41_715443fb55c29fd4a00fd2e9afbe3938 is 50, key is test_row_0/A:col10/1731564999981/Put/seqid=0 2024-11-14T06:16:40,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-14T06:16:40,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:40,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. as already flushing 2024-11-14T06:16:40,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:40,074 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:40,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:40,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:40,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-14T06:16:40,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741955_1131 (size=12154) 2024-11-14T06:16:40,079 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:40,085 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411141433d3e589634021b628c96595e9fc41_715443fb55c29fd4a00fd2e9afbe3938 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411141433d3e589634021b628c96595e9fc41_715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:40,087 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/e080682261094c1ba373cbbc33efe5a2, store: [table=TestAcidGuarantees family=A region=715443fb55c29fd4a00fd2e9afbe3938] 2024-11-14T06:16:40,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/e080682261094c1ba373cbbc33efe5a2 is 175, key is test_row_0/A:col10/1731564999981/Put/seqid=0 2024-11-14T06:16:40,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:40,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43476 deadline: 1731565060088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:40,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:40,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:40,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:40,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:40,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43446 deadline: 1731565060101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:40,117 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:40,117 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:40,117 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:40,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:40,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:40,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43444 deadline: 1731565060098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:40,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:40,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43470 deadline: 1731565060103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:40,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43484 deadline: 1731565060103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:40,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:40,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:40,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:40,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:40,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:40,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:40,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:40,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:40,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:40,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741956_1132 (size=30955) 2024-11-14T06:16:40,137 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/e080682261094c1ba373cbbc33efe5a2 2024-11-14T06:16:40,155 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/570f6541134d4cf99bfd4f820cfa9856 is 50, key is test_row_0/B:col10/1731564999981/Put/seqid=0 2024-11-14T06:16:40,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:40,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43476 deadline: 1731565060217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:40,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:40,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:40,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:40,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741957_1133 (size=12001) 2024-11-14T06:16:40,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-14T06:16:40,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:40,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. as already flushing 2024-11-14T06:16:40,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:40,231 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:40,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:40,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:40,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:40,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43484 deadline: 1731565060228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:40,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:40,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43446 deadline: 1731565060228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:40,243 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:40,243 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:40,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:40,243 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:40,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:40,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:40,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:40,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43470 deadline: 1731565060228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:40,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:40,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:40,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:40,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:40,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43444 deadline: 1731565060228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:40,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:40,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:40,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:40,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-14T06:16:40,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:40,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. as already flushing 2024-11-14T06:16:40,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:40,391 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:40,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:40,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:40,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:40,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43476 deadline: 1731565060427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:40,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:40,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:40,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:40,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:40,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43446 deadline: 1731565060447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:40,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:40,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43484 deadline: 1731565060447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:40,473 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:40,473 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:40,473 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:40,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:40,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:40,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:40,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:40,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43470 deadline: 1731565060448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:40,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:40,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43444 deadline: 1731565060460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:40,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:40,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:40,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:40,484 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:40,484 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:40,484 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:40,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-14T06:16:40,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:40,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. as already flushing 2024-11-14T06:16:40,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:40,546 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:40,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:40,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:40,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-14T06:16:40,628 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/570f6541134d4cf99bfd4f820cfa9856 2024-11-14T06:16:40,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/158a1a8358b24f05abb46c68d13d6439 is 50, key is test_row_0/C:col10/1731564999981/Put/seqid=0 2024-11-14T06:16:40,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741958_1134 (size=12001) 2024-11-14T06:16:40,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-14T06:16:40,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:40,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. as already flushing 2024-11-14T06:16:40,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:40,702 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:40,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:40,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:40,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:40,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43476 deadline: 1731565060737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:40,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:40,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:40,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:40,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:40,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43446 deadline: 1731565060777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:40,779 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:40,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:40,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:40,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:40,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43484 deadline: 1731565060778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:40,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:40,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:40,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:40,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:40,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43444 deadline: 1731565060788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:40,797 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:40,797 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:40,797 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:40,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:40,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43470 deadline: 1731565060789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:40,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:40,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:40,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:40,856 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-14T06:16:40,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:40,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. as already flushing 2024-11-14T06:16:40,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:40,857 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:40,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:40,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:40,976 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T06:16:41,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-14T06:16:41,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:41,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. as already flushing 2024-11-14T06:16:41,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:41,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:41,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:41,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:41,076 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/158a1a8358b24f05abb46c68d13d6439 2024-11-14T06:16:41,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/e080682261094c1ba373cbbc33efe5a2 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/e080682261094c1ba373cbbc33efe5a2 2024-11-14T06:16:41,090 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/e080682261094c1ba373cbbc33efe5a2, entries=150, sequenceid=53, filesize=30.2 K 2024-11-14T06:16:41,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/570f6541134d4cf99bfd4f820cfa9856 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/570f6541134d4cf99bfd4f820cfa9856 2024-11-14T06:16:41,116 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/570f6541134d4cf99bfd4f820cfa9856, entries=150, sequenceid=53, filesize=11.7 K 2024-11-14T06:16:41,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/158a1a8358b24f05abb46c68d13d6439 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/158a1a8358b24f05abb46c68d13d6439 2024-11-14T06:16:41,124 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/158a1a8358b24f05abb46c68d13d6439, entries=150, sequenceid=53, filesize=11.7 K 2024-11-14T06:16:41,125 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~105.11 KB/107630, heapSize ~297.23 KB/304368, currentSize=80.51 KB/82440 for 715443fb55c29fd4a00fd2e9afbe3938 in 1137ms, sequenceid=53, compaction requested=false 2024-11-14T06:16:41,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 715443fb55c29fd4a00fd2e9afbe3938: 2024-11-14T06:16:41,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-14T06:16:41,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:41,170 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 715443fb55c29fd4a00fd2e9afbe3938 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-14T06:16:41,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 715443fb55c29fd4a00fd2e9afbe3938, store=A 2024-11-14T06:16:41,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:41,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 715443fb55c29fd4a00fd2e9afbe3938, store=B 2024-11-14T06:16:41,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:41,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 715443fb55c29fd4a00fd2e9afbe3938, store=C 2024-11-14T06:16:41,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:41,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114f3d8ec31f2d648a4b2f5ca4e52e40fba_715443fb55c29fd4a00fd2e9afbe3938 is 50, key is test_row_0/A:col10/1731565000100/Put/seqid=0 2024-11-14T06:16:41,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. as already flushing 2024-11-14T06:16:41,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on 715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:41,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741959_1135 (size=12154) 2024-11-14T06:16:41,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:41,287 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114f3d8ec31f2d648a4b2f5ca4e52e40fba_715443fb55c29fd4a00fd2e9afbe3938 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f3d8ec31f2d648a4b2f5ca4e52e40fba_715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:41,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/3afd502297d24df58dfb5254939d0da1, store: [table=TestAcidGuarantees family=A region=715443fb55c29fd4a00fd2e9afbe3938] 2024-11-14T06:16:41,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/3afd502297d24df58dfb5254939d0da1 is 175, key is test_row_0/A:col10/1731565000100/Put/seqid=0 2024-11-14T06:16:41,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741960_1136 (size=30955) 2024-11-14T06:16:41,325 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=68, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/3afd502297d24df58dfb5254939d0da1 2024-11-14T06:16:41,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/4ba1ef576f2c46c49dbc434640d141ee is 50, key is test_row_0/B:col10/1731565000100/Put/seqid=0 2024-11-14T06:16:41,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:41,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43476 deadline: 1731565061363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:41,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:41,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43446 deadline: 1731565061364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:41,379 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:41,379 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:41,380 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:41,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:41,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:41,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:41,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741961_1137 (size=12001) 2024-11-14T06:16:41,386 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/4ba1ef576f2c46c49dbc434640d141ee 2024-11-14T06:16:41,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:41,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43470 deadline: 1731565061374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:41,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:41,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43484 deadline: 1731565061376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:41,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:41,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:41,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:41,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:41,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43444 deadline: 1731565061380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:41,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:41,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:41,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:41,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:41,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:41,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:41,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/7d4fb4ed667749768523fe2e43285241 is 50, key is test_row_0/C:col10/1731565000100/Put/seqid=0 2024-11-14T06:16:41,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741962_1138 (size=12001) 2024-11-14T06:16:41,491 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/7d4fb4ed667749768523fe2e43285241 2024-11-14T06:16:41,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:41,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43446 deadline: 1731565061487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:41,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:41,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43476 deadline: 1731565061487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:41,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:41,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43484 deadline: 1731565061498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:41,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:41,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43444 deadline: 1731565061498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:41,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:41,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:41,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:41,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:41,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:41,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:41,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:41,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:41,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:41,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:41,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43470 deadline: 1731565061498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:41,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:41,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:41,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:41,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:41,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:41,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:41,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/3afd502297d24df58dfb5254939d0da1 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/3afd502297d24df58dfb5254939d0da1 2024-11-14T06:16:41,522 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/3afd502297d24df58dfb5254939d0da1, entries=150, sequenceid=68, filesize=30.2 K 2024-11-14T06:16:41,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/4ba1ef576f2c46c49dbc434640d141ee as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/4ba1ef576f2c46c49dbc434640d141ee 2024-11-14T06:16:41,535 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/4ba1ef576f2c46c49dbc434640d141ee, entries=150, sequenceid=68, filesize=11.7 K 2024-11-14T06:16:41,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/7d4fb4ed667749768523fe2e43285241 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/7d4fb4ed667749768523fe2e43285241 2024-11-14T06:16:41,548 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/7d4fb4ed667749768523fe2e43285241, entries=150, sequenceid=68, filesize=11.7 K 2024-11-14T06:16:41,550 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 715443fb55c29fd4a00fd2e9afbe3938 in 380ms, sequenceid=68, compaction requested=true 2024-11-14T06:16:41,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 715443fb55c29fd4a00fd2e9afbe3938: 2024-11-14T06:16:41,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:41,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-14T06:16:41,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-14T06:16:41,556 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-14T06:16:41,556 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1050 sec 2024-11-14T06:16:41,559 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 2.1140 sec 2024-11-14T06:16:41,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-14T06:16:41,608 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:16:41,609 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T06:16:41,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-14T06:16:41,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T06:16:41,613 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T06:16:41,614 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T06:16:41,614 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T06:16:41,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T06:16:41,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on 715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:41,739 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 715443fb55c29fd4a00fd2e9afbe3938 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-14T06:16:41,743 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731565001355, max timestamp=1731565001738Num uniques 859; 2024-11-14T06:16:41,743 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T06:16:41,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 715443fb55c29fd4a00fd2e9afbe3938, store=A 2024-11-14T06:16:41,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:16:41,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 715443fb55c29fd4a00fd2e9afbe3938, store=B 2024-11-14T06:16:41,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:16:41,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 715443fb55c29fd4a00fd2e9afbe3938, store=C 2024-11-14T06:16:41,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:16:41,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T06:16:41,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:41,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. as already flushing 2024-11-14T06:16:41,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:41,769 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:41,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:41,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:41,779 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411146d8840f859dd4d4cbcb4d02606381b4d_715443fb55c29fd4a00fd2e9afbe3938 is 50, key is test_row_0/A:col10/1731565001730/Put/seqid=0 2024-11-14T06:16:41,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:41,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43446 deadline: 1731565061781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:41,792 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:41,792 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:41,792 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:41,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:41,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43476 deadline: 1731565061785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:41,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:41,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:41,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:41,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741963_1139 (size=12154) 2024-11-14T06:16:41,816 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:41,823 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411146d8840f859dd4d4cbcb4d02606381b4d_715443fb55c29fd4a00fd2e9afbe3938 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411146d8840f859dd4d4cbcb4d02606381b4d_715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:41,825 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/c3f337220a094bf58176e836fd60fa24, store: [table=TestAcidGuarantees family=A region=715443fb55c29fd4a00fd2e9afbe3938] 2024-11-14T06:16:41,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/c3f337220a094bf58176e836fd60fa24 is 175, key is test_row_0/A:col10/1731565001730/Put/seqid=0 2024-11-14T06:16:41,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:41,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43444 deadline: 1731565061792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:41,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:41,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:41,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:41,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:41,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43470 deadline: 1731565061793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:41,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:41,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43484 deadline: 1731565061790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:41,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:41,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:41,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:41,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:41,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:41,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:41,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741964_1140 (size=30955) 2024-11-14T06:16:41,871 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=89, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/c3f337220a094bf58176e836fd60fa24 2024-11-14T06:16:41,895 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/bbc27a327f054037a469722c424abc82 is 50, key is test_row_0/B:col10/1731565001730/Put/seqid=0 2024-11-14T06:16:41,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:41,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43446 deadline: 1731565061898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:41,918 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:41,918 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:41,918 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:41,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741965_1141 (size=12001) 2024-11-14T06:16:41,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T06:16:41,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:41,927 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/bbc27a327f054037a469722c424abc82 2024-11-14T06:16:41,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:41,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. as already flushing 2024-11-14T06:16:41,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:41,927 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:41,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:41,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T06:16:41,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43476 deadline: 1731565061918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:41,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:41,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:41,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:41,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:41,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/e6c73f57204d4c0d9ae715d1c8120d55 is 50, key is test_row_0/C:col10/1731565001730/Put/seqid=0 2024-11-14T06:16:41,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:41,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43444 deadline: 1731565061939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:41,954 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:41,954 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:41,954 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:41,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:41,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43484 deadline: 1731565061939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:41,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:41,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:41,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:41,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:41,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43470 deadline: 1731565061939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:41,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:41,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:41,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:42,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741966_1142 (size=12001) 2024-11-14T06:16:42,021 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/e6c73f57204d4c0d9ae715d1c8120d55 2024-11-14T06:16:42,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/c3f337220a094bf58176e836fd60fa24 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/c3f337220a094bf58176e836fd60fa24 2024-11-14T06:16:42,061 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/c3f337220a094bf58176e836fd60fa24, entries=150, sequenceid=89, filesize=30.2 K 2024-11-14T06:16:42,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/bbc27a327f054037a469722c424abc82 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/bbc27a327f054037a469722c424abc82 2024-11-14T06:16:42,076 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/bbc27a327f054037a469722c424abc82, entries=150, sequenceid=89, filesize=11.7 K 2024-11-14T06:16:42,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T06:16:42,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:42,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. as already flushing 2024-11-14T06:16:42,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:42,081 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:42,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:42,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:42,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/e6c73f57204d4c0d9ae715d1c8120d55 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/e6c73f57204d4c0d9ae715d1c8120d55 2024-11-14T06:16:42,094 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/e6c73f57204d4c0d9ae715d1c8120d55, entries=150, sequenceid=89, filesize=11.7 K 2024-11-14T06:16:42,122 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~272.22 KB/278752, currentSize=80.51 KB/82440 for 715443fb55c29fd4a00fd2e9afbe3938 in 383ms, sequenceid=89, compaction requested=true 2024-11-14T06:16:42,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 715443fb55c29fd4a00fd2e9afbe3938: 2024-11-14T06:16:42,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 715443fb55c29fd4a00fd2e9afbe3938:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T06:16:42,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:16:42,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 715443fb55c29fd4a00fd2e9afbe3938:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T06:16:42,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T06:16:42,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 715443fb55c29fd4a00fd2e9afbe3938:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T06:16:42,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-14T06:16:42,123 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T06:16:42,123 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T06:16:42,138 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 715443fb55c29fd4a00fd2e9afbe3938 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-14T06:16:42,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 715443fb55c29fd4a00fd2e9afbe3938, store=A 2024-11-14T06:16:42,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:42,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 715443fb55c29fd4a00fd2e9afbe3938, store=B 2024-11-14T06:16:42,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:42,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 715443fb55c29fd4a00fd2e9afbe3938, store=C 2024-11-14T06:16:42,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:42,140 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T06:16:42,140 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): 715443fb55c29fd4a00fd2e9afbe3938/C is initiating minor compaction (all files) 2024-11-14T06:16:42,140 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 715443fb55c29fd4a00fd2e9afbe3938/C in TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:42,141 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/612597fe630b42ce89c04d714f8754ff, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/158a1a8358b24f05abb46c68d13d6439, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/7d4fb4ed667749768523fe2e43285241, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/e6c73f57204d4c0d9ae715d1c8120d55] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp, totalSize=46.9 K 2024-11-14T06:16:42,141 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T06:16:42,141 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): 715443fb55c29fd4a00fd2e9afbe3938/A is initiating minor compaction (all files) 2024-11-14T06:16:42,141 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 715443fb55c29fd4a00fd2e9afbe3938/A in TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:42,141 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/a1186be964b14f66a3b84dfe4966754c, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/e080682261094c1ba373cbbc33efe5a2, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/3afd502297d24df58dfb5254939d0da1, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/c3f337220a094bf58176e836fd60fa24] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp, totalSize=137.7 K 2024-11-14T06:16:42,141 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:42,142 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. files: [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/a1186be964b14f66a3b84dfe4966754c, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/e080682261094c1ba373cbbc33efe5a2, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/3afd502297d24df58dfb5254939d0da1, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/c3f337220a094bf58176e836fd60fa24] 2024-11-14T06:16:42,142 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 612597fe630b42ce89c04d714f8754ff, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731564999463 2024-11-14T06:16:42,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on 715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:42,144 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1186be964b14f66a3b84dfe4966754c, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731564999443 2024-11-14T06:16:42,145 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 158a1a8358b24f05abb46c68d13d6439, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731564999662 2024-11-14T06:16:42,147 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting e080682261094c1ba373cbbc33efe5a2, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731564999662 2024-11-14T06:16:42,150 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d4fb4ed667749768523fe2e43285241, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731565000065 2024-11-14T06:16:42,150 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3afd502297d24df58dfb5254939d0da1, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731565000065 2024-11-14T06:16:42,151 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting e6c73f57204d4c0d9ae715d1c8120d55, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731565001730 2024-11-14T06:16:42,151 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3f337220a094bf58176e836fd60fa24, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731565001730 2024-11-14T06:16:42,202 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=715443fb55c29fd4a00fd2e9afbe3938] 2024-11-14T06:16:42,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T06:16:42,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:42,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. as already flushing 2024-11-14T06:16:42,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:42,237 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:42,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:42,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T06:16:42,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:42,243 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 715443fb55c29fd4a00fd2e9afbe3938#C#compaction#124 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:16:42,244 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/d1ec0ff934334b7fbf544a2f1a0be94d is 50, key is test_row_0/C:col10/1731565001730/Put/seqid=0 2024-11-14T06:16:42,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:42,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43484 deadline: 1731565062207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:42,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:42,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:42,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:42,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43470 deadline: 1731565062211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:42,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:42,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:42,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43446 deadline: 1731565062215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:42,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:42,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43444 deadline: 1731565062217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:42,259 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114cddd3d0462a243f3a7179a9f2b3492dd_715443fb55c29fd4a00fd2e9afbe3938 is 50, key is test_row_0/A:col10/1731565001781/Put/seqid=0 2024-11-14T06:16:42,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:42,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:42,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:42,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:42,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:42,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:42,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:42,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:42,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:42,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:42,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43476 deadline: 1731565062258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:42,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:42,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:42,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:42,289 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114fba33d58f8354dcebc6e823fcc900948_715443fb55c29fd4a00fd2e9afbe3938 store=[table=TestAcidGuarantees family=A region=715443fb55c29fd4a00fd2e9afbe3938] 2024-11-14T06:16:42,300 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114fba33d58f8354dcebc6e823fcc900948_715443fb55c29fd4a00fd2e9afbe3938, store=[table=TestAcidGuarantees family=A region=715443fb55c29fd4a00fd2e9afbe3938] 2024-11-14T06:16:42,301 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114fba33d58f8354dcebc6e823fcc900948_715443fb55c29fd4a00fd2e9afbe3938 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=715443fb55c29fd4a00fd2e9afbe3938] 2024-11-14T06:16:42,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741967_1143 (size=12139) 2024-11-14T06:16:42,376 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/d1ec0ff934334b7fbf544a2f1a0be94d as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/d1ec0ff934334b7fbf544a2f1a0be94d 2024-11-14T06:16:42,384 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 715443fb55c29fd4a00fd2e9afbe3938/C of 715443fb55c29fd4a00fd2e9afbe3938 into d1ec0ff934334b7fbf544a2f1a0be94d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:16:42,384 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 715443fb55c29fd4a00fd2e9afbe3938: 2024-11-14T06:16:42,384 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., storeName=715443fb55c29fd4a00fd2e9afbe3938/C, priority=12, startTime=1731565002123; duration=0sec 2024-11-14T06:16:42,384 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:16:42,384 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 715443fb55c29fd4a00fd2e9afbe3938:C 2024-11-14T06:16:42,384 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T06:16:42,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:42,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43484 deadline: 1731565062367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:42,387 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T06:16:42,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:42,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:42,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:42,388 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): 715443fb55c29fd4a00fd2e9afbe3938/B is initiating minor compaction (all files) 2024-11-14T06:16:42,388 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 715443fb55c29fd4a00fd2e9afbe3938/B in TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:42,388 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/9af36e19d06342168b217f9b8d0bcad4, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/570f6541134d4cf99bfd4f820cfa9856, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/4ba1ef576f2c46c49dbc434640d141ee, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/bbc27a327f054037a469722c424abc82] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp, totalSize=46.9 K 2024-11-14T06:16:42,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741968_1144 (size=14594) 2024-11-14T06:16:42,388 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 9af36e19d06342168b217f9b8d0bcad4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731564999463 2024-11-14T06:16:42,389 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 570f6541134d4cf99bfd4f820cfa9856, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731564999662 2024-11-14T06:16:42,389 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ba1ef576f2c46c49dbc434640d141ee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731565000065 2024-11-14T06:16:42,390 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:42,390 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting bbc27a327f054037a469722c424abc82, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731565001730 2024-11-14T06:16:42,395 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114cddd3d0462a243f3a7179a9f2b3492dd_715443fb55c29fd4a00fd2e9afbe3938 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114cddd3d0462a243f3a7179a9f2b3492dd_715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:42,397 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/af394cc2cd3f465895eff9b7ec29a1a1, store: [table=TestAcidGuarantees family=A region=715443fb55c29fd4a00fd2e9afbe3938] 2024-11-14T06:16:42,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/af394cc2cd3f465895eff9b7ec29a1a1 is 175, key is test_row_0/A:col10/1731565001781/Put/seqid=0 2024-11-14T06:16:42,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T06:16:42,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:42,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. as already flushing 2024-11-14T06:16:42,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:42,407 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:42,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:42,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741969_1145 (size=4469) 2024-11-14T06:16:42,411 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 715443fb55c29fd4a00fd2e9afbe3938#A#compaction#123 average throughput is 0.12 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:16:42,412 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/3b72a3fab8e845bd93c373afd5535c3d is 175, key is test_row_0/A:col10/1731565001730/Put/seqid=0 2024-11-14T06:16:42,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:42,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:42,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43470 deadline: 1731565062389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:42,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:42,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:42,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:42,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:42,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43446 deadline: 1731565062389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:42,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:42,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:42,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:42,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:42,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43444 deadline: 1731565062389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:42,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:42,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:42,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:42,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:42,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43476 deadline: 1731565062398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:42,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:42,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:42,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:42,439 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 715443fb55c29fd4a00fd2e9afbe3938#B#compaction#126 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:16:42,440 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/871fea2b6b9946d3a8abcd94260e55b2 is 50, key is test_row_0/B:col10/1731565001730/Put/seqid=0 2024-11-14T06:16:42,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741970_1146 (size=39549) 2024-11-14T06:16:42,475 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=105, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/af394cc2cd3f465895eff9b7ec29a1a1 2024-11-14T06:16:42,514 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/cdd582fbf177434b84f4ee173f9099d5 is 50, key is test_row_0/B:col10/1731565001781/Put/seqid=0 2024-11-14T06:16:42,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741972_1148 (size=12139) 2024-11-14T06:16:42,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T06:16:42,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:42,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. as already flushing 2024-11-14T06:16:42,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:42,570 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:42,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:42,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:42,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741971_1147 (size=31093) 2024-11-14T06:16:42,585 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/3b72a3fab8e845bd93c373afd5535c3d as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/3b72a3fab8e845bd93c373afd5535c3d 2024-11-14T06:16:42,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741973_1149 (size=12001) 2024-11-14T06:16:42,592 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/cdd582fbf177434b84f4ee173f9099d5 2024-11-14T06:16:42,596 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 715443fb55c29fd4a00fd2e9afbe3938/A of 715443fb55c29fd4a00fd2e9afbe3938 into 3b72a3fab8e845bd93c373afd5535c3d(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:16:42,596 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 715443fb55c29fd4a00fd2e9afbe3938: 2024-11-14T06:16:42,596 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., storeName=715443fb55c29fd4a00fd2e9afbe3938/A, priority=12, startTime=1731565002122; duration=0sec 2024-11-14T06:16:42,597 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:16:42,597 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 715443fb55c29fd4a00fd2e9afbe3938:A 2024-11-14T06:16:42,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:42,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43484 deadline: 1731565062598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:42,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:42,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:42,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:42,616 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/cd956313276a486ba4fa604f322abce2 is 50, key is test_row_0/C:col10/1731565001781/Put/seqid=0 2024-11-14T06:16:42,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:42,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43470 deadline: 1731565062628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:42,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:42,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:42,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:42,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:42,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43446 deadline: 1731565062628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:42,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:42,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43476 deadline: 1731565062628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:42,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:42,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43444 deadline: 1731565062628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:42,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:42,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:42,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:42,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:42,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:42,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:42,665 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:42,665 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:42,665 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:42,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741974_1150 (size=12001) 2024-11-14T06:16:42,675 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/cd956313276a486ba4fa604f322abce2 2024-11-14T06:16:42,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/af394cc2cd3f465895eff9b7ec29a1a1 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/af394cc2cd3f465895eff9b7ec29a1a1 2024-11-14T06:16:42,710 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/af394cc2cd3f465895eff9b7ec29a1a1, entries=200, sequenceid=105, filesize=38.6 K 2024-11-14T06:16:42,711 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/cdd582fbf177434b84f4ee173f9099d5 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/cdd582fbf177434b84f4ee173f9099d5 2024-11-14T06:16:42,723 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/cdd582fbf177434b84f4ee173f9099d5, entries=150, sequenceid=105, filesize=11.7 K 2024-11-14T06:16:42,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T06:16:42,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:42,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/cd956313276a486ba4fa604f322abce2 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/cd956313276a486ba4fa604f322abce2 2024-11-14T06:16:42,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. as already flushing 2024-11-14T06:16:42,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:42,735 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:42,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:42,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:42,746 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/cd956313276a486ba4fa604f322abce2, entries=150, sequenceid=105, filesize=11.7 K 2024-11-14T06:16:42,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T06:16:42,748 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 715443fb55c29fd4a00fd2e9afbe3938 in 610ms, sequenceid=105, compaction requested=false 2024-11-14T06:16:42,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 715443fb55c29fd4a00fd2e9afbe3938: 2024-11-14T06:16:42,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T06:16:42,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:42,890 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 715443fb55c29fd4a00fd2e9afbe3938 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-14T06:16:42,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 715443fb55c29fd4a00fd2e9afbe3938, store=A 2024-11-14T06:16:42,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:42,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 715443fb55c29fd4a00fd2e9afbe3938, store=B 2024-11-14T06:16:42,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:42,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 715443fb55c29fd4a00fd2e9afbe3938, store=C 2024-11-14T06:16:42,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:42,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on 715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:42,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. as already flushing 2024-11-14T06:16:42,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111423a7cb7c8b99456498c2d6a6279ebbe7_715443fb55c29fd4a00fd2e9afbe3938 is 50, key is test_row_0/A:col10/1731565002184/Put/seqid=0 2024-11-14T06:16:42,955 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/871fea2b6b9946d3a8abcd94260e55b2 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/871fea2b6b9946d3a8abcd94260e55b2 2024-11-14T06:16:42,967 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 715443fb55c29fd4a00fd2e9afbe3938/B of 715443fb55c29fd4a00fd2e9afbe3938 into 871fea2b6b9946d3a8abcd94260e55b2(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:16:42,967 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 715443fb55c29fd4a00fd2e9afbe3938: 2024-11-14T06:16:42,967 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., storeName=715443fb55c29fd4a00fd2e9afbe3938/B, priority=12, startTime=1731565002123; duration=0sec 2024-11-14T06:16:42,968 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:16:42,968 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 715443fb55c29fd4a00fd2e9afbe3938:B 2024-11-14T06:16:42,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741975_1151 (size=12154) 2024-11-14T06:16:42,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:42,977 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111423a7cb7c8b99456498c2d6a6279ebbe7_715443fb55c29fd4a00fd2e9afbe3938 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111423a7cb7c8b99456498c2d6a6279ebbe7_715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:42,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/eccf45b0738b40419ff61477c4a08cc8, store: [table=TestAcidGuarantees family=A region=715443fb55c29fd4a00fd2e9afbe3938] 2024-11-14T06:16:42,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/eccf45b0738b40419ff61477c4a08cc8 is 175, key is test_row_0/A:col10/1731565002184/Put/seqid=0 2024-11-14T06:16:42,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741976_1152 (size=30955) 2024-11-14T06:16:42,985 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=123, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/eccf45b0738b40419ff61477c4a08cc8 2024-11-14T06:16:42,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:42,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43444 deadline: 1731565062979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:42,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:42,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:42,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:42,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:42,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43446 deadline: 1731565062982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:42,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:42,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:42,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43470 deadline: 1731565062985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:42,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:42,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:42,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:42,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:42,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:42,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:42,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43476 deadline: 1731565062977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:42,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:42,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:42,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:43,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:43,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43484 deadline: 1731565062991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:43,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:43,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:43,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:43,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/4bacb134f948428582d5e9b1d05c6e85 is 50, key is test_row_0/B:col10/1731565002184/Put/seqid=0 2024-11-14T06:16:43,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741977_1153 (size=12001) 2024-11-14T06:16:43,085 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/4bacb134f948428582d5e9b1d05c6e85 2024-11-14T06:16:43,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/a7395012eb8d47cc8e9a0eb0271b8b94 is 50, key is test_row_0/C:col10/1731565002184/Put/seqid=0 2024-11-14T06:16:43,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:43,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43444 deadline: 1731565063098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:43,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:43,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:43,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:43,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:43,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43446 deadline: 1731565063099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:43,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:43,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43470 deadline: 1731565063099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:43,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:43,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:43,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:43,106 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:43,106 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:43,106 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:43,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:43,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43476 deadline: 1731565063107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:43,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:43,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43484 deadline: 1731565063107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:43,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:43,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:43,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:43,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:43,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:43,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:43,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741978_1154 (size=12001) 2024-11-14T06:16:43,115 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/a7395012eb8d47cc8e9a0eb0271b8b94 2024-11-14T06:16:43,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/eccf45b0738b40419ff61477c4a08cc8 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/eccf45b0738b40419ff61477c4a08cc8 2024-11-14T06:16:43,130 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/eccf45b0738b40419ff61477c4a08cc8, entries=150, sequenceid=123, filesize=30.2 K 2024-11-14T06:16:43,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/4bacb134f948428582d5e9b1d05c6e85 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/4bacb134f948428582d5e9b1d05c6e85 2024-11-14T06:16:43,146 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/4bacb134f948428582d5e9b1d05c6e85, entries=150, sequenceid=123, filesize=11.7 K 2024-11-14T06:16:43,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/a7395012eb8d47cc8e9a0eb0271b8b94 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/a7395012eb8d47cc8e9a0eb0271b8b94 2024-11-14T06:16:43,152 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/a7395012eb8d47cc8e9a0eb0271b8b94, entries=150, sequenceid=123, filesize=11.7 K 2024-11-14T06:16:43,153 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 715443fb55c29fd4a00fd2e9afbe3938 in 263ms, sequenceid=123, compaction requested=true 2024-11-14T06:16:43,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 715443fb55c29fd4a00fd2e9afbe3938: 2024-11-14T06:16:43,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:43,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-14T06:16:43,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-14T06:16:43,163 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-14T06:16:43,163 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5440 sec 2024-11-14T06:16:43,165 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.5550 sec 2024-11-14T06:16:43,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on 715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:43,310 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 715443fb55c29fd4a00fd2e9afbe3938 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-14T06:16:43,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 715443fb55c29fd4a00fd2e9afbe3938, store=A 2024-11-14T06:16:43,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:43,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 715443fb55c29fd4a00fd2e9afbe3938, store=B 2024-11-14T06:16:43,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:43,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 715443fb55c29fd4a00fd2e9afbe3938, store=C 2024-11-14T06:16:43,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:43,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111444837ad1420342d9ae77679297bdd30f_715443fb55c29fd4a00fd2e9afbe3938 is 50, key is test_row_0/A:col10/1731565003308/Put/seqid=0 2024-11-14T06:16:43,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:43,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43484 deadline: 1731565063332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:43,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:43,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:43,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:43,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:43,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43476 deadline: 1731565063340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:43,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:43,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43446 deadline: 1731565063345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:43,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:43,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43444 deadline: 1731565063350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:43,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:43,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:43,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:43,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:43,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:43,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:43,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:43,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:43,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:43,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:43,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43470 deadline: 1731565063354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:43,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:43,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:43,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:43,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741979_1155 (size=14794) 2024-11-14T06:16:43,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:43,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43484 deadline: 1731565063468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:43,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:43,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:43,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:43,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:43,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:43,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43444 deadline: 1731565063469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:43,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43476 deadline: 1731565063469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:43,475 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:43,475 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:43,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:43,475 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:43,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43446 deadline: 1731565063469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:43,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:43,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:43,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:43,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:43,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:43,476 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:43,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:43,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43470 deadline: 1731565063477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:43,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:43,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:43,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:43,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:43,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43484 deadline: 1731565063680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:43,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:43,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:43,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:43,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:43,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43444 deadline: 1731565063687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:43,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:43,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:43,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43476 deadline: 1731565063688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:43,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:43,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:43,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:43,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:43,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:43,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:43,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43446 deadline: 1731565063691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:43,697 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:43,697 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:43,697 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:43,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:43,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43470 deadline: 1731565063697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:43,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:43,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:43,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:43,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T06:16:43,758 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:16:43,758 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T06:16:43,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-14T06:16:43,760 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T06:16:43,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T06:16:43,772 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T06:16:43,772 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T06:16:43,811 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:43,816 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111444837ad1420342d9ae77679297bdd30f_715443fb55c29fd4a00fd2e9afbe3938 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111444837ad1420342d9ae77679297bdd30f_715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:43,817 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/59a52cb27b844c28880ce9269b0f5179, store: [table=TestAcidGuarantees family=A region=715443fb55c29fd4a00fd2e9afbe3938] 2024-11-14T06:16:43,818 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/59a52cb27b844c28880ce9269b0f5179 is 175, key is test_row_0/A:col10/1731565003308/Put/seqid=0 2024-11-14T06:16:43,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741980_1156 (size=39749) 2024-11-14T06:16:43,838 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=141, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/59a52cb27b844c28880ce9269b0f5179 2024-11-14T06:16:43,851 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/793a7e323d31487cb0299407353153cf is 50, key is test_row_0/B:col10/1731565003308/Put/seqid=0 2024-11-14T06:16:43,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T06:16:43,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741981_1157 (size=12151) 2024-11-14T06:16:43,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T06:16:43,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:43,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. as already flushing 2024-11-14T06:16:43,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:43,924 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:43,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:43,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:44,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:44,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43484 deadline: 1731565063997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:44,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:44,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:44,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:44,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:44,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43446 deadline: 1731565064008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:44,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:44,009 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:44,009 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:44,009 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:44,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43470 deadline: 1731565064008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:44,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:44,010 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:44,010 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:44,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43444 deadline: 1731565064008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:44,010 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:44,010 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:44,010 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:44,010 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:44,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:44,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43476 deadline: 1731565064011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:44,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:44,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:44,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:44,077 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T06:16:44,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:44,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T06:16:44,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. as already flushing 2024-11-14T06:16:44,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:44,078 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:44,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:44,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:44,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T06:16:44,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:44,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. as already flushing 2024-11-14T06:16:44,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:44,230 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:44,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:44,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:44,286 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/793a7e323d31487cb0299407353153cf 2024-11-14T06:16:44,299 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/4d6bec6b538746739bcde75e94f0e448 is 50, key is test_row_0/C:col10/1731565003308/Put/seqid=0 2024-11-14T06:16:44,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741982_1158 (size=12151) 2024-11-14T06:16:44,326 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/4d6bec6b538746739bcde75e94f0e448 2024-11-14T06:16:44,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/59a52cb27b844c28880ce9269b0f5179 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/59a52cb27b844c28880ce9269b0f5179 2024-11-14T06:16:44,345 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/59a52cb27b844c28880ce9269b0f5179, entries=200, sequenceid=141, filesize=38.8 K 2024-11-14T06:16:44,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/793a7e323d31487cb0299407353153cf as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/793a7e323d31487cb0299407353153cf 2024-11-14T06:16:44,363 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/793a7e323d31487cb0299407353153cf, entries=150, sequenceid=141, filesize=11.9 K 2024-11-14T06:16:44,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/4d6bec6b538746739bcde75e94f0e448 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/4d6bec6b538746739bcde75e94f0e448 2024-11-14T06:16:44,372 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/4d6bec6b538746739bcde75e94f0e448, entries=150, sequenceid=141, filesize=11.9 K 2024-11-14T06:16:44,374 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 715443fb55c29fd4a00fd2e9afbe3938 in 1064ms, sequenceid=141, compaction requested=true 2024-11-14T06:16:44,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 715443fb55c29fd4a00fd2e9afbe3938: 2024-11-14T06:16:44,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 715443fb55c29fd4a00fd2e9afbe3938:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T06:16:44,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:16:44,374 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T06:16:44,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 715443fb55c29fd4a00fd2e9afbe3938:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T06:16:44,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:16:44,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 715443fb55c29fd4a00fd2e9afbe3938:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T06:16:44,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T06:16:44,374 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T06:16:44,375 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141346 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T06:16:44,376 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): 715443fb55c29fd4a00fd2e9afbe3938/A is initiating minor compaction (all files) 2024-11-14T06:16:44,376 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 715443fb55c29fd4a00fd2e9afbe3938/A in TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:44,376 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/3b72a3fab8e845bd93c373afd5535c3d, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/af394cc2cd3f465895eff9b7ec29a1a1, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/eccf45b0738b40419ff61477c4a08cc8, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/59a52cb27b844c28880ce9269b0f5179] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp, totalSize=138.0 K 2024-11-14T06:16:44,376 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:44,376 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. files: [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/3b72a3fab8e845bd93c373afd5535c3d, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/af394cc2cd3f465895eff9b7ec29a1a1, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/eccf45b0738b40419ff61477c4a08cc8, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/59a52cb27b844c28880ce9269b0f5179] 2024-11-14T06:16:44,376 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48292 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T06:16:44,376 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): 715443fb55c29fd4a00fd2e9afbe3938/B is initiating minor compaction (all files) 2024-11-14T06:16:44,377 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 715443fb55c29fd4a00fd2e9afbe3938/B in TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:44,377 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/871fea2b6b9946d3a8abcd94260e55b2, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/cdd582fbf177434b84f4ee173f9099d5, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/4bacb134f948428582d5e9b1d05c6e85, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/793a7e323d31487cb0299407353153cf] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp, totalSize=47.2 K 2024-11-14T06:16:44,377 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b72a3fab8e845bd93c373afd5535c3d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731565001730 2024-11-14T06:16:44,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T06:16:44,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T06:16:44,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:44,389 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 871fea2b6b9946d3a8abcd94260e55b2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731565001730 2024-11-14T06:16:44,389 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 715443fb55c29fd4a00fd2e9afbe3938 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-14T06:16:44,389 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting af394cc2cd3f465895eff9b7ec29a1a1, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731565001764 2024-11-14T06:16:44,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 715443fb55c29fd4a00fd2e9afbe3938, store=A 2024-11-14T06:16:44,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:44,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 715443fb55c29fd4a00fd2e9afbe3938, store=B 2024-11-14T06:16:44,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:44,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 715443fb55c29fd4a00fd2e9afbe3938, store=C 2024-11-14T06:16:44,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:44,389 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting cdd582fbf177434b84f4ee173f9099d5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731565001781 2024-11-14T06:16:44,389 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting eccf45b0738b40419ff61477c4a08cc8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1731565002184 2024-11-14T06:16:44,390 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bacb134f948428582d5e9b1d05c6e85, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1731565002184 2024-11-14T06:16:44,390 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 793a7e323d31487cb0299407353153cf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731565002981 2024-11-14T06:16:44,390 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59a52cb27b844c28880ce9269b0f5179, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731565002974 2024-11-14T06:16:44,421 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=715443fb55c29fd4a00fd2e9afbe3938] 2024-11-14T06:16:44,440 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 715443fb55c29fd4a00fd2e9afbe3938#B#compaction#136 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:16:44,441 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/ff82186fe43445339060ad17cd8d170a is 50, key is test_row_0/B:col10/1731565003308/Put/seqid=0 2024-11-14T06:16:44,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114d62ec2557e064134842ceafd4c6b6969_715443fb55c29fd4a00fd2e9afbe3938 is 50, key is test_row_0/A:col10/1731565003330/Put/seqid=0 2024-11-14T06:16:44,445 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411147bd1675677b640febf743ea0a062888a_715443fb55c29fd4a00fd2e9afbe3938 store=[table=TestAcidGuarantees family=A region=715443fb55c29fd4a00fd2e9afbe3938] 2024-11-14T06:16:44,451 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411147bd1675677b640febf743ea0a062888a_715443fb55c29fd4a00fd2e9afbe3938, store=[table=TestAcidGuarantees family=A region=715443fb55c29fd4a00fd2e9afbe3938] 2024-11-14T06:16:44,451 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411147bd1675677b640febf743ea0a062888a_715443fb55c29fd4a00fd2e9afbe3938 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=715443fb55c29fd4a00fd2e9afbe3938] 2024-11-14T06:16:44,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741983_1159 (size=12425) 2024-11-14T06:16:44,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741984_1160 (size=12304) 2024-11-14T06:16:44,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:44,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741985_1161 (size=4469) 2024-11-14T06:16:44,493 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 715443fb55c29fd4a00fd2e9afbe3938#A#compaction#135 average throughput is 0.34 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:16:44,494 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/4de44b33c21644e489e25dfccf3fdf9f is 175, key is test_row_0/A:col10/1731565003308/Put/seqid=0 2024-11-14T06:16:44,494 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114d62ec2557e064134842ceafd4c6b6969_715443fb55c29fd4a00fd2e9afbe3938 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d62ec2557e064134842ceafd4c6b6969_715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:44,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/2ae0adcccfc84d2c8fb526f727a47b4f, store: [table=TestAcidGuarantees family=A region=715443fb55c29fd4a00fd2e9afbe3938] 2024-11-14T06:16:44,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/2ae0adcccfc84d2c8fb526f727a47b4f is 175, key is test_row_0/A:col10/1731565003330/Put/seqid=0 2024-11-14T06:16:44,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on 715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:44,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. as already flushing 2024-11-14T06:16:44,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741986_1162 (size=31379) 2024-11-14T06:16:44,528 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/4de44b33c21644e489e25dfccf3fdf9f as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/4de44b33c21644e489e25dfccf3fdf9f 2024-11-14T06:16:44,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741987_1163 (size=31105) 2024-11-14T06:16:44,539 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 715443fb55c29fd4a00fd2e9afbe3938/A of 715443fb55c29fd4a00fd2e9afbe3938 into 4de44b33c21644e489e25dfccf3fdf9f(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:16:44,539 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 715443fb55c29fd4a00fd2e9afbe3938: 2024-11-14T06:16:44,539 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., storeName=715443fb55c29fd4a00fd2e9afbe3938/A, priority=12, startTime=1731565004374; duration=0sec 2024-11-14T06:16:44,539 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:16:44,539 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 715443fb55c29fd4a00fd2e9afbe3938:A 2024-11-14T06:16:44,539 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T06:16:44,542 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48292 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T06:16:44,542 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): 715443fb55c29fd4a00fd2e9afbe3938/C is initiating minor compaction (all files) 2024-11-14T06:16:44,542 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 715443fb55c29fd4a00fd2e9afbe3938/C in TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:44,542 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/d1ec0ff934334b7fbf544a2f1a0be94d, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/cd956313276a486ba4fa604f322abce2, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/a7395012eb8d47cc8e9a0eb0271b8b94, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/4d6bec6b538746739bcde75e94f0e448] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp, totalSize=47.2 K 2024-11-14T06:16:44,542 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1ec0ff934334b7fbf544a2f1a0be94d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731565001730 2024-11-14T06:16:44,543 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd956313276a486ba4fa604f322abce2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731565001781 2024-11-14T06:16:44,543 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7395012eb8d47cc8e9a0eb0271b8b94, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1731565002184 2024-11-14T06:16:44,546 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d6bec6b538746739bcde75e94f0e448, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731565002981 2024-11-14T06:16:44,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:44,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:44,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43476 deadline: 1731565064541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:44,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43470 deadline: 1731565064539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:44,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:44,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:44,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:44,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:44,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:44,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:44,561 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 715443fb55c29fd4a00fd2e9afbe3938#C#compaction#138 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:16:44,561 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/1f44eb6b28294eec8c0f1c42a1ec061d is 50, key is test_row_0/C:col10/1731565003308/Put/seqid=0 2024-11-14T06:16:44,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:44,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43484 deadline: 1731565064555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:44,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:44,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43444 deadline: 1731565064555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:44,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:44,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43446 deadline: 1731565064553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:44,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:44,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:44,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:44,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:44,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:44,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:44,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:44,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:44,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:44,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741988_1164 (size=12425) 2024-11-14T06:16:44,626 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/1f44eb6b28294eec8c0f1c42a1ec061d as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/1f44eb6b28294eec8c0f1c42a1ec061d 2024-11-14T06:16:44,640 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 715443fb55c29fd4a00fd2e9afbe3938/C of 715443fb55c29fd4a00fd2e9afbe3938 into 1f44eb6b28294eec8c0f1c42a1ec061d(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:16:44,640 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 715443fb55c29fd4a00fd2e9afbe3938: 2024-11-14T06:16:44,640 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., storeName=715443fb55c29fd4a00fd2e9afbe3938/C, priority=12, startTime=1731565004374; duration=0sec 2024-11-14T06:16:44,641 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:16:44,641 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 715443fb55c29fd4a00fd2e9afbe3938:C 2024-11-14T06:16:44,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:44,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43476 deadline: 1731565064661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:44,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:44,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43470 deadline: 1731565064661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:44,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:44,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:44,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:44,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:44,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:44,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:44,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:44,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43444 deadline: 1731565064667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:44,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:44,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:44,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43484 deadline: 1731565064667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:44,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:44,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:44,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:44,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:44,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:44,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:44,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43446 deadline: 1731565064668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:44,674 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:44,674 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:44,674 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:44,864 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/ff82186fe43445339060ad17cd8d170a as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/ff82186fe43445339060ad17cd8d170a 2024-11-14T06:16:44,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:44,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43470 deadline: 1731565064869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:44,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:44,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:44,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:44,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:44,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43476 deadline: 1731565064869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:44,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:44,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:44,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:44,878 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 715443fb55c29fd4a00fd2e9afbe3938/B of 715443fb55c29fd4a00fd2e9afbe3938 into ff82186fe43445339060ad17cd8d170a(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:16:44,878 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 715443fb55c29fd4a00fd2e9afbe3938: 2024-11-14T06:16:44,878 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., storeName=715443fb55c29fd4a00fd2e9afbe3938/B, priority=12, startTime=1731565004374; duration=0sec 2024-11-14T06:16:44,878 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:16:44,878 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 715443fb55c29fd4a00fd2e9afbe3938:B 2024-11-14T06:16:44,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:44,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:44,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43484 deadline: 1731565064877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:44,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:44,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:44,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:44,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:44,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43446 deadline: 1731565064877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:44,883 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:44,883 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:44,883 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:44,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43444 deadline: 1731565064877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:44,885 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:44,885 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:44,885 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:44,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T06:16:44,942 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=157, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/2ae0adcccfc84d2c8fb526f727a47b4f 2024-11-14T06:16:44,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/bc625f4f6a374c499b96ff5c4d96eb5e is 50, key is test_row_0/B:col10/1731565003330/Put/seqid=0 2024-11-14T06:16:45,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741989_1165 (size=12151) 2024-11-14T06:16:45,019 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/bc625f4f6a374c499b96ff5c4d96eb5e 2024-11-14T06:16:45,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/468a5c60281540ab9df8098b39c535af is 50, key is test_row_0/C:col10/1731565003330/Put/seqid=0 2024-11-14T06:16:45,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741990_1166 (size=12151) 2024-11-14T06:16:45,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:45,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43470 deadline: 1731565065178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:45,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:45,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43476 deadline: 1731565065178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:45,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:45,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43446 deadline: 1731565065188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:45,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:45,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:45,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:45,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:45,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43484 deadline: 1731565065188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:45,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:45,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:45,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:45,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:45,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:45,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:45,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43444 deadline: 1731565065188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:45,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:45,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:45,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:45,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:45,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:45,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:45,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:45,491 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/468a5c60281540ab9df8098b39c535af 2024-11-14T06:16:45,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/2ae0adcccfc84d2c8fb526f727a47b4f as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/2ae0adcccfc84d2c8fb526f727a47b4f 2024-11-14T06:16:45,505 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/2ae0adcccfc84d2c8fb526f727a47b4f, entries=150, sequenceid=157, filesize=30.4 K 2024-11-14T06:16:45,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/bc625f4f6a374c499b96ff5c4d96eb5e as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/bc625f4f6a374c499b96ff5c4d96eb5e 2024-11-14T06:16:45,531 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/bc625f4f6a374c499b96ff5c4d96eb5e, entries=150, sequenceid=157, filesize=11.9 K 2024-11-14T06:16:45,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/468a5c60281540ab9df8098b39c535af as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/468a5c60281540ab9df8098b39c535af 2024-11-14T06:16:45,539 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-14T06:16:45,539 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-14T06:16:45,548 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/468a5c60281540ab9df8098b39c535af, entries=150, sequenceid=157, filesize=11.9 K 2024-11-14T06:16:45,550 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 715443fb55c29fd4a00fd2e9afbe3938 in 1160ms, sequenceid=157, compaction requested=false 2024-11-14T06:16:45,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 715443fb55c29fd4a00fd2e9afbe3938: 2024-11-14T06:16:45,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:45,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-14T06:16:45,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-14T06:16:45,553 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-14T06:16:45,554 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7790 sec 2024-11-14T06:16:45,556 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.7960 sec 2024-11-14T06:16:45,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on 715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:45,703 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 715443fb55c29fd4a00fd2e9afbe3938 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-14T06:16:45,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 715443fb55c29fd4a00fd2e9afbe3938, store=A 2024-11-14T06:16:45,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:45,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 715443fb55c29fd4a00fd2e9afbe3938, store=B 2024-11-14T06:16:45,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:45,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 715443fb55c29fd4a00fd2e9afbe3938, store=C 2024-11-14T06:16:45,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:45,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411140148c56a7dd64fb987d6768c57c2fc6f_715443fb55c29fd4a00fd2e9afbe3938 is 50, key is test_row_0/A:col10/1731565004552/Put/seqid=0 2024-11-14T06:16:45,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:45,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43444 deadline: 1731565065723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:45,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:45,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43476 deadline: 1731565065724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:45,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:45,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:45,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:45,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:45,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43484 deadline: 1731565065726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:45,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:45,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:45,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:45,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:45,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:45,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:45,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:45,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43446 deadline: 1731565065729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:45,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:45,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43470 deadline: 1731565065730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:45,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:45,745 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:45,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:45,745 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:45,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:45,745 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:45,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741991_1167 (size=12304) 2024-11-14T06:16:45,755 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:45,769 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411140148c56a7dd64fb987d6768c57c2fc6f_715443fb55c29fd4a00fd2e9afbe3938 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411140148c56a7dd64fb987d6768c57c2fc6f_715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:45,773 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/826cc3fd68de438d8c2dbc0f7fb39040, store: [table=TestAcidGuarantees family=A region=715443fb55c29fd4a00fd2e9afbe3938] 2024-11-14T06:16:45,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/826cc3fd68de438d8c2dbc0f7fb39040 is 175, key is test_row_0/A:col10/1731565004552/Put/seqid=0 2024-11-14T06:16:45,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741992_1168 (size=31105) 2024-11-14T06:16:45,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:45,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43476 deadline: 1731565065839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:45,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:45,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:45,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:45,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43444 deadline: 1731565065839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:45,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:45,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:45,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43484 deadline: 1731565065839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:45,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:45,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:45,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:45,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:45,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:45,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:45,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:45,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43470 deadline: 1731565065847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:45,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:45,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:45,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:45,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:45,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43446 deadline: 1731565065847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:45,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:45,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:45,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:45,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T06:16:45,907 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:16:45,907 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T06:16:45,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-14T06:16:45,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-14T06:16:45,912 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T06:16:45,913 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T06:16:45,914 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T06:16:46,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-14T06:16:46,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:46,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:46,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43484 deadline: 1731565066049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:46,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43476 deadline: 1731565066049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:46,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:46,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:46,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:46,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:46,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:46,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:46,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:46,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43470 deadline: 1731565066060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:46,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:46,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:46,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43446 deadline: 1731565066062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:46,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43444 deadline: 1731565066063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:46,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:46,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:46,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:46,068 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:46,068 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:46,068 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:46,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:46,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:46,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:46,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-14T06:16:46,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:46,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. as already flushing 2024-11-14T06:16:46,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:46,075 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:46,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:46,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:46,204 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=180, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/826cc3fd68de438d8c2dbc0f7fb39040 2024-11-14T06:16:46,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-14T06:16:46,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-14T06:16:46,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:46,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. as already flushing 2024-11-14T06:16:46,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:46,231 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:46,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:46,234 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/3c20446a1bdd48999bf2ef2a58556c78 is 50, key is test_row_0/B:col10/1731565004552/Put/seqid=0 2024-11-14T06:16:46,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:46,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741993_1169 (size=12151) 2024-11-14T06:16:46,256 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/3c20446a1bdd48999bf2ef2a58556c78 2024-11-14T06:16:46,293 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/1b6fbbae30e24052a5643719e54b5c2c is 50, key is test_row_0/C:col10/1731565004552/Put/seqid=0 2024-11-14T06:16:46,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741994_1170 (size=12151) 2024-11-14T06:16:46,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:46,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43476 deadline: 1731565066368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:46,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:46,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43484 deadline: 1731565066370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:46,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:46,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:46,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:46,379 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:46,379 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:46,379 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:46,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-14T06:16:46,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:46,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. as already flushing 2024-11-14T06:16:46,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:46,388 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:46,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:46,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:46,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:46,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43444 deadline: 1731565066387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:46,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:46,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:46,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:46,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:46,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43446 deadline: 1731565066387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:46,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:46,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43470 deadline: 1731565066388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:46,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:46,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:46,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:46,400 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:46,400 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:46,400 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:46,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-14T06:16:46,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-14T06:16:46,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:46,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. as already flushing 2024-11-14T06:16:46,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:46,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:46,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:46,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:46,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-14T06:16:46,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:46,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. as already flushing 2024-11-14T06:16:46,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:46,701 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:46,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:46,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:46,732 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/1b6fbbae30e24052a5643719e54b5c2c 2024-11-14T06:16:46,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/826cc3fd68de438d8c2dbc0f7fb39040 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/826cc3fd68de438d8c2dbc0f7fb39040 2024-11-14T06:16:46,753 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/826cc3fd68de438d8c2dbc0f7fb39040, entries=150, sequenceid=180, filesize=30.4 K 2024-11-14T06:16:46,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/3c20446a1bdd48999bf2ef2a58556c78 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/3c20446a1bdd48999bf2ef2a58556c78 2024-11-14T06:16:46,761 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/3c20446a1bdd48999bf2ef2a58556c78, entries=150, sequenceid=180, filesize=11.9 K 2024-11-14T06:16:46,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/1b6fbbae30e24052a5643719e54b5c2c as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/1b6fbbae30e24052a5643719e54b5c2c 2024-11-14T06:16:46,767 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/1b6fbbae30e24052a5643719e54b5c2c, entries=150, sequenceid=180, filesize=11.9 K 2024-11-14T06:16:46,770 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 715443fb55c29fd4a00fd2e9afbe3938 in 1067ms, sequenceid=180, compaction requested=true 2024-11-14T06:16:46,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 715443fb55c29fd4a00fd2e9afbe3938: 2024-11-14T06:16:46,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 715443fb55c29fd4a00fd2e9afbe3938:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T06:16:46,771 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:16:46,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:16:46,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 715443fb55c29fd4a00fd2e9afbe3938:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T06:16:46,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:16:46,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 715443fb55c29fd4a00fd2e9afbe3938:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T06:16:46,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T06:16:46,772 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:16:46,772 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93589 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:16:46,772 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): 715443fb55c29fd4a00fd2e9afbe3938/A is initiating minor compaction (all files) 2024-11-14T06:16:46,772 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 715443fb55c29fd4a00fd2e9afbe3938/A in TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:46,772 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/4de44b33c21644e489e25dfccf3fdf9f, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/2ae0adcccfc84d2c8fb526f727a47b4f, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/826cc3fd68de438d8c2dbc0f7fb39040] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp, totalSize=91.4 K 2024-11-14T06:16:46,773 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:46,773 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. files: [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/4de44b33c21644e489e25dfccf3fdf9f, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/2ae0adcccfc84d2c8fb526f727a47b4f, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/826cc3fd68de438d8c2dbc0f7fb39040] 2024-11-14T06:16:46,773 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4de44b33c21644e489e25dfccf3fdf9f, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731565002981 2024-11-14T06:16:46,774 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ae0adcccfc84d2c8fb526f727a47b4f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731565003327 2024-11-14T06:16:46,774 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 826cc3fd68de438d8c2dbc0f7fb39040, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731565004552 2024-11-14T06:16:46,777 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:16:46,777 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): 715443fb55c29fd4a00fd2e9afbe3938/B is initiating minor compaction (all files) 2024-11-14T06:16:46,777 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 715443fb55c29fd4a00fd2e9afbe3938/B in TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:46,777 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/ff82186fe43445339060ad17cd8d170a, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/bc625f4f6a374c499b96ff5c4d96eb5e, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/3c20446a1bdd48999bf2ef2a58556c78] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp, totalSize=35.9 K 2024-11-14T06:16:46,778 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting ff82186fe43445339060ad17cd8d170a, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731565002981 2024-11-14T06:16:46,779 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting bc625f4f6a374c499b96ff5c4d96eb5e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731565003327 2024-11-14T06:16:46,779 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c20446a1bdd48999bf2ef2a58556c78, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731565004552 2024-11-14T06:16:46,785 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=715443fb55c29fd4a00fd2e9afbe3938] 2024-11-14T06:16:46,798 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114cedcf11a43fa4d2590a8fc4fd666b952_715443fb55c29fd4a00fd2e9afbe3938 store=[table=TestAcidGuarantees family=A region=715443fb55c29fd4a00fd2e9afbe3938] 2024-11-14T06:16:46,802 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114cedcf11a43fa4d2590a8fc4fd666b952_715443fb55c29fd4a00fd2e9afbe3938, store=[table=TestAcidGuarantees family=A region=715443fb55c29fd4a00fd2e9afbe3938] 2024-11-14T06:16:46,803 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114cedcf11a43fa4d2590a8fc4fd666b952_715443fb55c29fd4a00fd2e9afbe3938 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=715443fb55c29fd4a00fd2e9afbe3938] 2024-11-14T06:16:46,805 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 715443fb55c29fd4a00fd2e9afbe3938#B#compaction#145 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:16:46,806 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/a099012ff74b4710b32aee290970471a is 50, key is test_row_0/B:col10/1731565004552/Put/seqid=0 2024-11-14T06:16:46,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741995_1171 (size=4469) 2024-11-14T06:16:46,831 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 715443fb55c29fd4a00fd2e9afbe3938#A#compaction#144 average throughput is 0.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:16:46,832 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/fc3c0546dc37475c992790b9785b136f is 175, key is test_row_0/A:col10/1731565004552/Put/seqid=0 2024-11-14T06:16:46,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-14T06:16:46,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:46,857 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 715443fb55c29fd4a00fd2e9afbe3938 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-14T06:16:46,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 715443fb55c29fd4a00fd2e9afbe3938, store=A 2024-11-14T06:16:46,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:46,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 715443fb55c29fd4a00fd2e9afbe3938, store=B 2024-11-14T06:16:46,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:46,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 715443fb55c29fd4a00fd2e9afbe3938, store=C 2024-11-14T06:16:46,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:46,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741996_1172 (size=12527) 2024-11-14T06:16:46,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. as already flushing 2024-11-14T06:16:46,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on 715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:46,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741997_1173 (size=31481) 2024-11-14T06:16:46,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411149aaa17081542465b9ef5e4c8cdfb0701_715443fb55c29fd4a00fd2e9afbe3938 is 50, key is test_row_0/A:col10/1731565005708/Put/seqid=0 2024-11-14T06:16:46,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:46,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43444 deadline: 1731565066954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:46,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:46,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:46,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:46,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:46,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43446 deadline: 1731565066955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:46,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:46,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:46,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:46,961 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731565006946, max timestamp=1731565006958Num uniques 859; 2024-11-14T06:16:46,961 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T06:16:46,962 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731565006946, max timestamp=1731565006958Num uniques 859; 2024-11-14T06:16:46,962 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T06:16:46,964 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731565006946, max timestamp=1731565006958Num uniques 859; 2024-11-14T06:16:46,964 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T06:16:46,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741998_1174 (size=12304) 2024-11-14T06:16:47,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:47,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43470 deadline: 1731565067024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:47,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:47,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:47,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:47,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:47,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43476 deadline: 1731565067026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:47,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:47,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:47,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:47,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:47,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43484 deadline: 1731565067033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:47,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:47,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:47,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:47,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-14T06:16:47,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:47,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43444 deadline: 1731565067067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:47,076 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:47,076 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:47,076 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:47,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:47,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43446 deadline: 1731565067067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:47,084 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:47,084 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:47,084 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:47,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:47,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43470 deadline: 1731565067138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:47,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:47,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:47,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43476 deadline: 1731565067138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:47,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:47,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:47,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:47,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:47,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:47,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:47,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43484 deadline: 1731565067147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:47,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:47,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:47,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:47,277 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/a099012ff74b4710b32aee290970471a as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/a099012ff74b4710b32aee290970471a 2024-11-14T06:16:47,287 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 715443fb55c29fd4a00fd2e9afbe3938/B of 715443fb55c29fd4a00fd2e9afbe3938 into a099012ff74b4710b32aee290970471a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:16:47,287 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 715443fb55c29fd4a00fd2e9afbe3938: 2024-11-14T06:16:47,287 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., storeName=715443fb55c29fd4a00fd2e9afbe3938/B, priority=13, startTime=1731565006771; duration=0sec 2024-11-14T06:16:47,287 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:16:47,287 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 715443fb55c29fd4a00fd2e9afbe3938:B 2024-11-14T06:16:47,287 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:16:47,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:47,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43444 deadline: 1731565067287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:47,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:47,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:47,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:47,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:47,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43446 deadline: 1731565067288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:47,289 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:16:47,289 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): 715443fb55c29fd4a00fd2e9afbe3938/C is initiating minor compaction (all files) 2024-11-14T06:16:47,289 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:47,289 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 715443fb55c29fd4a00fd2e9afbe3938/C in TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:47,289 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:47,289 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:47,289 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/1f44eb6b28294eec8c0f1c42a1ec061d, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/468a5c60281540ab9df8098b39c535af, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/1b6fbbae30e24052a5643719e54b5c2c] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp, totalSize=35.9 K 2024-11-14T06:16:47,290 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f44eb6b28294eec8c0f1c42a1ec061d, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731565002981 2024-11-14T06:16:47,291 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 468a5c60281540ab9df8098b39c535af, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731565003327 2024-11-14T06:16:47,292 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b6fbbae30e24052a5643719e54b5c2c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731565004552 2024-11-14T06:16:47,318 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/fc3c0546dc37475c992790b9785b136f as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/fc3c0546dc37475c992790b9785b136f 2024-11-14T06:16:47,324 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 715443fb55c29fd4a00fd2e9afbe3938/A of 715443fb55c29fd4a00fd2e9afbe3938 into fc3c0546dc37475c992790b9785b136f(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:16:47,325 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 715443fb55c29fd4a00fd2e9afbe3938: 2024-11-14T06:16:47,325 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., storeName=715443fb55c29fd4a00fd2e9afbe3938/A, priority=13, startTime=1731565006771; duration=0sec 2024-11-14T06:16:47,325 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:16:47,325 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 715443fb55c29fd4a00fd2e9afbe3938:A 2024-11-14T06:16:47,335 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 715443fb55c29fd4a00fd2e9afbe3938#C#compaction#147 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:16:47,335 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/856d4b6837954c2289094c681ecb9859 is 50, key is test_row_0/C:col10/1731565004552/Put/seqid=0 2024-11-14T06:16:47,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:47,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43470 deadline: 1731565067358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:47,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:47,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:47,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43484 deadline: 1731565067358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:47,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43476 deadline: 1731565067358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:47,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:47,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:47,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:47,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:47,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:47,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:47,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:47,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:47,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:47,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:47,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741999_1175 (size=12527) 2024-11-14T06:16:47,387 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411149aaa17081542465b9ef5e4c8cdfb0701_715443fb55c29fd4a00fd2e9afbe3938 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411149aaa17081542465b9ef5e4c8cdfb0701_715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:47,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/3f84bb2ea6534781837a9d75c76f811c, store: [table=TestAcidGuarantees family=A region=715443fb55c29fd4a00fd2e9afbe3938] 2024-11-14T06:16:47,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/3f84bb2ea6534781837a9d75c76f811c is 175, key is test_row_0/A:col10/1731565005708/Put/seqid=0 2024-11-14T06:16:47,393 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/856d4b6837954c2289094c681ecb9859 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/856d4b6837954c2289094c681ecb9859 2024-11-14T06:16:47,404 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 715443fb55c29fd4a00fd2e9afbe3938/C of 715443fb55c29fd4a00fd2e9afbe3938 into 856d4b6837954c2289094c681ecb9859(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:16:47,404 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 715443fb55c29fd4a00fd2e9afbe3938: 2024-11-14T06:16:47,404 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., storeName=715443fb55c29fd4a00fd2e9afbe3938/C, priority=13, startTime=1731565006771; duration=0sec 2024-11-14T06:16:47,404 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:16:47,404 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 715443fb55c29fd4a00fd2e9afbe3938:C 2024-11-14T06:16:47,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742000_1176 (size=31105) 2024-11-14T06:16:47,421 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=192, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/3f84bb2ea6534781837a9d75c76f811c 2024-11-14T06:16:47,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/e52448332b5c4f8db9ba761778e2cb5a is 50, key is test_row_0/B:col10/1731565005708/Put/seqid=0 2024-11-14T06:16:47,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742001_1177 (size=12151) 2024-11-14T06:16:47,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:47,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43446 deadline: 1731565067597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:47,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:47,605 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:47,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43444 deadline: 1731565067597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:47,605 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:47,605 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:47,607 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:47,607 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:47,607 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:47,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:47,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43476 deadline: 1731565067678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:47,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:47,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:47,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:47,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:47,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43484 deadline: 1731565067678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:47,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:47,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:47,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:47,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:47,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43470 deadline: 1731565067678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:47,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:47,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:47,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:47,917 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/e52448332b5c4f8db9ba761778e2cb5a 2024-11-14T06:16:47,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/fc3023e3117d438f844458bd03e375d5 is 50, key is test_row_0/C:col10/1731565005708/Put/seqid=0 2024-11-14T06:16:47,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742002_1178 (size=12151) 2024-11-14T06:16:48,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-14T06:16:48,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:48,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43446 deadline: 1731565068117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:48,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:48,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:48,122 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:48,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:48,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43444 deadline: 1731565068117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:48,123 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:48,123 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:48,123 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:48,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:48,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43470 deadline: 1731565068198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:48,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:48,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:48,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:48,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:48,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:48,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43484 deadline: 1731565068206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:48,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43476 deadline: 1731565068206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:48,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:48,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:48,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:48,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:48,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:48,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:48,397 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/fc3023e3117d438f844458bd03e375d5 2024-11-14T06:16:48,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/3f84bb2ea6534781837a9d75c76f811c as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/3f84bb2ea6534781837a9d75c76f811c 2024-11-14T06:16:48,434 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/3f84bb2ea6534781837a9d75c76f811c, entries=150, sequenceid=192, filesize=30.4 K 2024-11-14T06:16:48,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/e52448332b5c4f8db9ba761778e2cb5a as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/e52448332b5c4f8db9ba761778e2cb5a 2024-11-14T06:16:48,446 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/e52448332b5c4f8db9ba761778e2cb5a, entries=150, sequenceid=192, filesize=11.9 K 2024-11-14T06:16:48,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/fc3023e3117d438f844458bd03e375d5 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/fc3023e3117d438f844458bd03e375d5 2024-11-14T06:16:48,454 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/fc3023e3117d438f844458bd03e375d5, entries=150, sequenceid=192, filesize=11.9 K 2024-11-14T06:16:48,455 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=127.47 KB/130530 for 715443fb55c29fd4a00fd2e9afbe3938 in 1598ms, sequenceid=192, compaction requested=false 2024-11-14T06:16:48,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 715443fb55c29fd4a00fd2e9afbe3938: 2024-11-14T06:16:48,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:48,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-14T06:16:48,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-14T06:16:48,460 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-14T06:16:48,460 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5440 sec 2024-11-14T06:16:48,462 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.5540 sec 2024-11-14T06:16:49,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on 715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:49,142 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 715443fb55c29fd4a00fd2e9afbe3938 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-14T06:16:49,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 715443fb55c29fd4a00fd2e9afbe3938, store=A 2024-11-14T06:16:49,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:16:49,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 715443fb55c29fd4a00fd2e9afbe3938, store=B 2024-11-14T06:16:49,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:16:49,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 715443fb55c29fd4a00fd2e9afbe3938, store=C 2024-11-14T06:16:49,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:16:49,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411149cdea612b0ed4d68aa011a74c8246d8d_715443fb55c29fd4a00fd2e9afbe3938 is 50, key is test_row_0/A:col10/1731565007024/Put/seqid=0 2024-11-14T06:16:49,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742003_1179 (size=14794) 2024-11-14T06:16:49,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:49,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43470 deadline: 1731565069208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:49,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:49,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43476 deadline: 1731565069217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:49,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:49,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43484 deadline: 1731565069219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:49,233 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to ef93ef3a83c5,42283,1731564965945 for 'test_row_1' in 715443fb55c29fd4a00fd2e9afbe3938 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T06:16:49,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:49,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:49,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:49,234 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to ef93ef3a83c5,42283,1731564965945 for 'test_row_2' in 715443fb55c29fd4a00fd2e9afbe3938 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T06:16:49,233 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to ef93ef3a83c5,42283,1731564965945 for 'test_row_0' in 715443fb55c29fd4a00fd2e9afbe3938 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T06:16:49,235 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:49,235 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:49,235 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:49,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:49,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:49,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:49,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:49,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43444 deadline: 1731565069222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:49,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:49,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:49,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:49,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:49,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43446 deadline: 1731565069226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:49,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:49,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:49,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:49,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:49,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43444 deadline: 1731565069347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:49,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:49,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43446 deadline: 1731565069349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:49,355 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:49,355 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:49,355 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:49,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:49,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:49,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:49,490 INFO [master/ef93ef3a83c5:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-14T06:16:49,490 INFO [master/ef93ef3a83c5:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-14T06:16:49,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:49,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43446 deadline: 1731565069561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:49,572 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:49,572 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:49,572 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:49,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:49,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43444 deadline: 1731565069567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:49,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:49,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:49,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:49,597 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:49,603 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411149cdea612b0ed4d68aa011a74c8246d8d_715443fb55c29fd4a00fd2e9afbe3938 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411149cdea612b0ed4d68aa011a74c8246d8d_715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:49,605 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/07b1e010b3e84af389f12d64d5fd310e, store: [table=TestAcidGuarantees family=A region=715443fb55c29fd4a00fd2e9afbe3938] 2024-11-14T06:16:49,605 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/07b1e010b3e84af389f12d64d5fd310e is 175, key is test_row_0/A:col10/1731565007024/Put/seqid=0 2024-11-14T06:16:49,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742004_1180 (size=39749) 2024-11-14T06:16:49,663 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=230, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/07b1e010b3e84af389f12d64d5fd310e 2024-11-14T06:16:49,683 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/03192e1cec07493a82d21beee2406d04 is 50, key is test_row_0/B:col10/1731565007024/Put/seqid=0 2024-11-14T06:16:49,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742005_1181 (size=12151) 2024-11-14T06:16:49,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:49,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43446 deadline: 1731565069877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:49,884 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:49,884 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:49,884 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:49,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:49,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43444 deadline: 1731565069887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:49,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:49,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:49,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:50,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-14T06:16:50,088 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:16:50,093 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T06:16:50,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-14T06:16:50,095 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T06:16:50,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T06:16:50,101 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T06:16:50,101 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T06:16:50,145 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/03192e1cec07493a82d21beee2406d04 2024-11-14T06:16:50,171 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/a2f133a05d6346e89ea0f93be09c6136 is 50, key is test_row_0/C:col10/1731565007024/Put/seqid=0 2024-11-14T06:16:50,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T06:16:50,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742006_1182 (size=12151) 2024-11-14T06:16:50,219 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/a2f133a05d6346e89ea0f93be09c6136 2024-11-14T06:16:50,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/07b1e010b3e84af389f12d64d5fd310e as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/07b1e010b3e84af389f12d64d5fd310e 2024-11-14T06:16:50,235 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/07b1e010b3e84af389f12d64d5fd310e, entries=200, sequenceid=230, filesize=38.8 K 2024-11-14T06:16:50,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/03192e1cec07493a82d21beee2406d04 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/03192e1cec07493a82d21beee2406d04 2024-11-14T06:16:50,257 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T06:16:50,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:50,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. as already flushing 2024-11-14T06:16:50,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:50,258 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:50,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:50,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:50,264 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/03192e1cec07493a82d21beee2406d04, entries=150, sequenceid=230, filesize=11.9 K 2024-11-14T06:16:50,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/a2f133a05d6346e89ea0f93be09c6136 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/a2f133a05d6346e89ea0f93be09c6136 2024-11-14T06:16:50,275 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/a2f133a05d6346e89ea0f93be09c6136, entries=150, sequenceid=230, filesize=11.9 K 2024-11-14T06:16:50,276 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 715443fb55c29fd4a00fd2e9afbe3938 in 1134ms, sequenceid=230, compaction requested=true 2024-11-14T06:16:50,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 715443fb55c29fd4a00fd2e9afbe3938: 2024-11-14T06:16:50,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 715443fb55c29fd4a00fd2e9afbe3938:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T06:16:50,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:16:50,276 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:16:50,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 715443fb55c29fd4a00fd2e9afbe3938:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T06:16:50,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:16:50,276 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:16:50,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 715443fb55c29fd4a00fd2e9afbe3938:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T06:16:50,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:16:50,278 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:16:50,278 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102335 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:16:50,278 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): 715443fb55c29fd4a00fd2e9afbe3938/A is initiating minor compaction (all files) 2024-11-14T06:16:50,278 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): 715443fb55c29fd4a00fd2e9afbe3938/B is initiating minor compaction (all files) 2024-11-14T06:16:50,278 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 715443fb55c29fd4a00fd2e9afbe3938/A in TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:50,278 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 715443fb55c29fd4a00fd2e9afbe3938/B in TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:50,278 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/a099012ff74b4710b32aee290970471a, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/e52448332b5c4f8db9ba761778e2cb5a, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/03192e1cec07493a82d21beee2406d04] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp, totalSize=36.0 K 2024-11-14T06:16:50,278 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/fc3c0546dc37475c992790b9785b136f, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/3f84bb2ea6534781837a9d75c76f811c, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/07b1e010b3e84af389f12d64d5fd310e] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp, totalSize=99.9 K 2024-11-14T06:16:50,278 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:50,278 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. files: [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/fc3c0546dc37475c992790b9785b136f, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/3f84bb2ea6534781837a9d75c76f811c, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/07b1e010b3e84af389f12d64d5fd310e] 2024-11-14T06:16:50,279 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting a099012ff74b4710b32aee290970471a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731565004552 2024-11-14T06:16:50,279 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc3c0546dc37475c992790b9785b136f, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731565004552 2024-11-14T06:16:50,279 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting e52448332b5c4f8db9ba761778e2cb5a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731565005708 2024-11-14T06:16:50,279 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f84bb2ea6534781837a9d75c76f811c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731565005708 2024-11-14T06:16:50,280 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 03192e1cec07493a82d21beee2406d04, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731565006998 2024-11-14T06:16:50,280 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07b1e010b3e84af389f12d64d5fd310e, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731565006998 2024-11-14T06:16:50,304 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 715443fb55c29fd4a00fd2e9afbe3938#B#compaction#153 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:16:50,305 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/378d6ffc43f64f44bf9633954e9b6618 is 50, key is test_row_0/B:col10/1731565007024/Put/seqid=0 2024-11-14T06:16:50,318 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=715443fb55c29fd4a00fd2e9afbe3938] 2024-11-14T06:16:50,337 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111408c7156a7dfd47e1983b63382229cbda_715443fb55c29fd4a00fd2e9afbe3938 store=[table=TestAcidGuarantees family=A region=715443fb55c29fd4a00fd2e9afbe3938] 2024-11-14T06:16:50,340 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111408c7156a7dfd47e1983b63382229cbda_715443fb55c29fd4a00fd2e9afbe3938, store=[table=TestAcidGuarantees family=A region=715443fb55c29fd4a00fd2e9afbe3938] 2024-11-14T06:16:50,340 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111408c7156a7dfd47e1983b63382229cbda_715443fb55c29fd4a00fd2e9afbe3938 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=715443fb55c29fd4a00fd2e9afbe3938] 2024-11-14T06:16:50,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742007_1183 (size=12629) 2024-11-14T06:16:50,356 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/378d6ffc43f64f44bf9633954e9b6618 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/378d6ffc43f64f44bf9633954e9b6618 2024-11-14T06:16:50,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742008_1184 (size=4469) 2024-11-14T06:16:50,370 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 715443fb55c29fd4a00fd2e9afbe3938/B of 715443fb55c29fd4a00fd2e9afbe3938 into 378d6ffc43f64f44bf9633954e9b6618(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:16:50,370 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 715443fb55c29fd4a00fd2e9afbe3938: 2024-11-14T06:16:50,370 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., storeName=715443fb55c29fd4a00fd2e9afbe3938/B, priority=13, startTime=1731565010276; duration=0sec 2024-11-14T06:16:50,370 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:16:50,370 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 715443fb55c29fd4a00fd2e9afbe3938:B 2024-11-14T06:16:50,370 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 715443fb55c29fd4a00fd2e9afbe3938#A#compaction#154 average throughput is 0.47 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:16:50,371 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:16:50,371 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/4d5c5ba8aca640158244b85848068103 is 175, key is test_row_0/A:col10/1731565007024/Put/seqid=0 2024-11-14T06:16:50,372 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:16:50,375 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): 715443fb55c29fd4a00fd2e9afbe3938/C is initiating minor compaction (all files) 2024-11-14T06:16:50,376 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 715443fb55c29fd4a00fd2e9afbe3938/C in TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:50,376 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/856d4b6837954c2289094c681ecb9859, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/fc3023e3117d438f844458bd03e375d5, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/a2f133a05d6346e89ea0f93be09c6136] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp, totalSize=36.0 K 2024-11-14T06:16:50,377 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 856d4b6837954c2289094c681ecb9859, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731565004552 2024-11-14T06:16:50,377 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting fc3023e3117d438f844458bd03e375d5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731565005708 2024-11-14T06:16:50,378 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting a2f133a05d6346e89ea0f93be09c6136, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731565006998 2024-11-14T06:16:50,389 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 715443fb55c29fd4a00fd2e9afbe3938 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-14T06:16:50,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 715443fb55c29fd4a00fd2e9afbe3938, store=A 2024-11-14T06:16:50,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:50,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 715443fb55c29fd4a00fd2e9afbe3938, store=B 2024-11-14T06:16:50,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:50,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 715443fb55c29fd4a00fd2e9afbe3938, store=C 2024-11-14T06:16:50,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:50,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on 715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:50,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742009_1185 (size=31583) 2024-11-14T06:16:50,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T06:16:50,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:50,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. as already flushing 2024-11-14T06:16:50,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:50,411 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:50,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:50,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:50,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T06:16:50,424 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 715443fb55c29fd4a00fd2e9afbe3938#C#compaction#155 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:16:50,425 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/76bc5f78d39b4659bb535cac41635477 is 50, key is test_row_0/C:col10/1731565007024/Put/seqid=0 2024-11-14T06:16:50,427 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111406327244ce5643b9b1e305223a3b97c0_715443fb55c29fd4a00fd2e9afbe3938 is 50, key is test_row_0/A:col10/1731565010387/Put/seqid=0 2024-11-14T06:16:50,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742010_1186 (size=12629) 2024-11-14T06:16:50,483 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731565010443, max timestamp=1731565010476Num uniques 859; 2024-11-14T06:16:50,483 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T06:16:50,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731565010443, max timestamp=1731565010476Num uniques 859; 2024-11-14T06:16:50,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T06:16:50,490 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731565010443, max timestamp=1731565010476Num uniques 859; 2024-11-14T06:16:50,490 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T06:16:50,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742011_1187 (size=14794) 2024-11-14T06:16:50,493 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:50,498 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111406327244ce5643b9b1e305223a3b97c0_715443fb55c29fd4a00fd2e9afbe3938 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111406327244ce5643b9b1e305223a3b97c0_715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:50,502 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/36a6414b86334c3e89e0b4415de8719b, store: [table=TestAcidGuarantees family=A region=715443fb55c29fd4a00fd2e9afbe3938] 2024-11-14T06:16:50,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/36a6414b86334c3e89e0b4415de8719b is 175, key is test_row_0/A:col10/1731565010387/Put/seqid=0 2024-11-14T06:16:50,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742012_1188 (size=39749) 2024-11-14T06:16:50,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T06:16:50,570 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=242, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/36a6414b86334c3e89e0b4415de8719b 2024-11-14T06:16:50,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:50,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. as already flushing 2024-11-14T06:16:50,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:50,576 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:50,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:50,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:50,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/04d1ca038df349ed80deba946ba42ece is 50, key is test_row_0/B:col10/1731565010387/Put/seqid=0 2024-11-14T06:16:50,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:50,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43446 deadline: 1731565070583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:50,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:50,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:50,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:50,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:50,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43444 deadline: 1731565070587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:50,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:50,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:50,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:50,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742013_1189 (size=12151) 2024-11-14T06:16:50,626 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/04d1ca038df349ed80deba946ba42ece 2024-11-14T06:16:50,650 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/3b0a24b4ced14636a390de780dcd64f8 is 50, key is test_row_0/C:col10/1731565010387/Put/seqid=0 2024-11-14T06:16:50,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:50,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43446 deadline: 1731565070697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:50,701 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:50,701 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:50,701 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:50,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742014_1190 (size=12151) 2024-11-14T06:16:50,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:50,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43444 deadline: 1731565070708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:50,712 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/3b0a24b4ced14636a390de780dcd64f8 2024-11-14T06:16:50,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:50,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:50,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:50,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T06:16:50,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T06:16:50,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:50,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. as already flushing 2024-11-14T06:16:50,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:50,731 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:50,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:50,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:50,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/36a6414b86334c3e89e0b4415de8719b as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/36a6414b86334c3e89e0b4415de8719b 2024-11-14T06:16:50,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/36a6414b86334c3e89e0b4415de8719b, entries=200, sequenceid=242, filesize=38.8 K 2024-11-14T06:16:50,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/04d1ca038df349ed80deba946ba42ece as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/04d1ca038df349ed80deba946ba42ece 2024-11-14T06:16:50,797 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/04d1ca038df349ed80deba946ba42ece, entries=150, sequenceid=242, filesize=11.9 K 2024-11-14T06:16:50,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/3b0a24b4ced14636a390de780dcd64f8 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/3b0a24b4ced14636a390de780dcd64f8 2024-11-14T06:16:50,827 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/4d5c5ba8aca640158244b85848068103 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/4d5c5ba8aca640158244b85848068103 2024-11-14T06:16:50,837 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/3b0a24b4ced14636a390de780dcd64f8, entries=150, sequenceid=242, filesize=11.9 K 2024-11-14T06:16:50,839 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 715443fb55c29fd4a00fd2e9afbe3938 in 451ms, sequenceid=242, compaction requested=false 2024-11-14T06:16:50,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 715443fb55c29fd4a00fd2e9afbe3938: 2024-11-14T06:16:50,864 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 715443fb55c29fd4a00fd2e9afbe3938/A of 715443fb55c29fd4a00fd2e9afbe3938 into 4d5c5ba8aca640158244b85848068103(size=30.8 K), total size for store is 69.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:16:50,864 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 715443fb55c29fd4a00fd2e9afbe3938: 2024-11-14T06:16:50,864 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., storeName=715443fb55c29fd4a00fd2e9afbe3938/A, priority=13, startTime=1731565010276; duration=0sec 2024-11-14T06:16:50,864 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:16:50,864 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 715443fb55c29fd4a00fd2e9afbe3938:A 2024-11-14T06:16:50,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T06:16:50,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:50,886 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 715443fb55c29fd4a00fd2e9afbe3938 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-14T06:16:50,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 715443fb55c29fd4a00fd2e9afbe3938, store=A 2024-11-14T06:16:50,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:16:50,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 715443fb55c29fd4a00fd2e9afbe3938, store=B 2024-11-14T06:16:50,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:16:50,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 715443fb55c29fd4a00fd2e9afbe3938, store=C 2024-11-14T06:16:50,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:16:50,913 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/76bc5f78d39b4659bb535cac41635477 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/76bc5f78d39b4659bb535cac41635477 2024-11-14T06:16:50,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on 715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:50,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. as already flushing 2024-11-14T06:16:50,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114dcb2fd7a12af4c299e3b3eeffa525082_715443fb55c29fd4a00fd2e9afbe3938 is 50, key is test_row_0/A:col10/1731565010585/Put/seqid=0 2024-11-14T06:16:50,934 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 715443fb55c29fd4a00fd2e9afbe3938/C of 715443fb55c29fd4a00fd2e9afbe3938 into 76bc5f78d39b4659bb535cac41635477(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:16:50,934 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 715443fb55c29fd4a00fd2e9afbe3938: 2024-11-14T06:16:50,934 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., storeName=715443fb55c29fd4a00fd2e9afbe3938/C, priority=13, startTime=1731565010276; duration=0sec 2024-11-14T06:16:50,934 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:16:50,935 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 715443fb55c29fd4a00fd2e9afbe3938:C 2024-11-14T06:16:50,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742015_1191 (size=12454) 2024-11-14T06:16:51,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:51,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43446 deadline: 1731565070993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:51,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:51,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43444 deadline: 1731565070997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:51,006 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:51,006 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:51,006 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:51,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:51,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:51,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:51,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:51,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43446 deadline: 1731565071117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:51,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:51,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43444 deadline: 1731565071117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:51,123 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:51,123 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:51,123 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:51,123 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:51,123 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:51,123 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:51,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T06:16:51,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:51,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43484 deadline: 1731565071248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:51,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:51,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43476 deadline: 1731565071258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:51,259 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to ef93ef3a83c5,42283,1731564965945 for 'test_row_1' in 715443fb55c29fd4a00fd2e9afbe3938 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T06:16:51,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:51,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:51,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:51,260 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to ef93ef3a83c5,42283,1731564965945 for 'test_row_2' in 715443fb55c29fd4a00fd2e9afbe3938 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T06:16:51,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:51,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:51,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:51,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:51,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43470 deadline: 1731565071258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:51,266 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to ef93ef3a83c5,42283,1731564965945 for 'test_row_0' in 715443fb55c29fd4a00fd2e9afbe3938 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4242 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T06:16:51,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:51,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:51,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:51,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:51,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43446 deadline: 1731565071328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:51,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:51,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:51,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:51,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:51,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43444 deadline: 1731565071328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:51,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:51,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:51,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:51,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:51,378 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114dcb2fd7a12af4c299e3b3eeffa525082_715443fb55c29fd4a00fd2e9afbe3938 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114dcb2fd7a12af4c299e3b3eeffa525082_715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:51,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/12692f8cd5564659ae326bed7d000d87, store: [table=TestAcidGuarantees family=A region=715443fb55c29fd4a00fd2e9afbe3938] 2024-11-14T06:16:51,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/12692f8cd5564659ae326bed7d000d87 is 175, key is test_row_0/A:col10/1731565010585/Put/seqid=0 2024-11-14T06:16:51,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742016_1192 (size=31255) 2024-11-14T06:16:51,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:51,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43446 deadline: 1731565071637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:51,641 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:51,641 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:51,641 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:51,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:51,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43444 deadline: 1731565071648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:51,649 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:51,649 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:51,649 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:51,857 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=278, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/12692f8cd5564659ae326bed7d000d87 2024-11-14T06:16:51,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/acd7219ab0bd4bb98eadf3ce2cafe864 is 50, key is test_row_0/B:col10/1731565010585/Put/seqid=0 2024-11-14T06:16:51,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742017_1193 (size=12301) 2024-11-14T06:16:52,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:52,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43446 deadline: 1731565072148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:52,150 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:52,150 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:52,150 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:52,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:52,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43444 deadline: 1731565072157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:52,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:52,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:52,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:52,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T06:16:52,337 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/acd7219ab0bd4bb98eadf3ce2cafe864 2024-11-14T06:16:52,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/c75d2d2c006144b483a9de2fa6cee18d is 50, key is test_row_0/C:col10/1731565010585/Put/seqid=0 2024-11-14T06:16:52,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742018_1194 (size=12301) 2024-11-14T06:16:52,373 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/c75d2d2c006144b483a9de2fa6cee18d 2024-11-14T06:16:52,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/12692f8cd5564659ae326bed7d000d87 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/12692f8cd5564659ae326bed7d000d87 2024-11-14T06:16:52,390 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/12692f8cd5564659ae326bed7d000d87, entries=150, sequenceid=278, filesize=30.5 K 2024-11-14T06:16:52,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/acd7219ab0bd4bb98eadf3ce2cafe864 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/acd7219ab0bd4bb98eadf3ce2cafe864 2024-11-14T06:16:52,405 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/acd7219ab0bd4bb98eadf3ce2cafe864, entries=150, sequenceid=278, filesize=12.0 K 2024-11-14T06:16:52,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/c75d2d2c006144b483a9de2fa6cee18d as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/c75d2d2c006144b483a9de2fa6cee18d 2024-11-14T06:16:52,412 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/c75d2d2c006144b483a9de2fa6cee18d, entries=150, sequenceid=278, filesize=12.0 K 2024-11-14T06:16:52,425 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 715443fb55c29fd4a00fd2e9afbe3938 in 1538ms, sequenceid=278, compaction requested=true 2024-11-14T06:16:52,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 715443fb55c29fd4a00fd2e9afbe3938: 2024-11-14T06:16:52,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:52,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-14T06:16:52,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-14T06:16:52,428 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-14T06:16:52,428 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3250 sec 2024-11-14T06:16:52,435 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.3400 sec 2024-11-14T06:16:53,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on 715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:53,159 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 715443fb55c29fd4a00fd2e9afbe3938 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-14T06:16:53,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 715443fb55c29fd4a00fd2e9afbe3938, store=A 2024-11-14T06:16:53,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:53,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 715443fb55c29fd4a00fd2e9afbe3938, store=B 2024-11-14T06:16:53,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:53,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 715443fb55c29fd4a00fd2e9afbe3938, store=C 2024-11-14T06:16:53,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:53,170 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411146f1f116575484bbfb23dd710d399b0d8_715443fb55c29fd4a00fd2e9afbe3938 is 50, key is test_row_0/A:col10/1731565013158/Put/seqid=0 2024-11-14T06:16:53,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742019_1195 (size=14994) 2024-11-14T06:16:53,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:53,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43444 deadline: 1731565073213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:53,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:53,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:53,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:53,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:53,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43446 deadline: 1731565073214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:53,221 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:53,221 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:53,221 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:53,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:53,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43444 deadline: 1731565073327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:53,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:53,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:53,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:53,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:53,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43446 deadline: 1731565073327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:53,335 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:53,335 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:53,335 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:53,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:53,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43444 deadline: 1731565073540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:53,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:53,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43446 deadline: 1731565073540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:53,543 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:53,543 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:53,543 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:53,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:53,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:53,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:53,586 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:53,597 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411146f1f116575484bbfb23dd710d399b0d8_715443fb55c29fd4a00fd2e9afbe3938 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411146f1f116575484bbfb23dd710d399b0d8_715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:53,605 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/545a58322453491fab76a9bb5bc12c6f, store: [table=TestAcidGuarantees family=A region=715443fb55c29fd4a00fd2e9afbe3938] 2024-11-14T06:16:53,605 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/545a58322453491fab76a9bb5bc12c6f is 175, key is test_row_0/A:col10/1731565013158/Put/seqid=0 2024-11-14T06:16:53,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742020_1196 (size=39949) 2024-11-14T06:16:53,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:53,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43446 deadline: 1731565073848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:53,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:53,852 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:53,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43444 deadline: 1731565073848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:53,852 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:53,852 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:53,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:53,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:53,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:54,046 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=291, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/545a58322453491fab76a9bb5bc12c6f 2024-11-14T06:16:54,059 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/2c603ec86ba342c7b2f4c1d5ff698ccf is 50, key is test_row_0/B:col10/1731565013158/Put/seqid=0 2024-11-14T06:16:54,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742021_1197 (size=12301) 2024-11-14T06:16:54,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T06:16:54,278 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:16:54,278 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T06:16:54,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-14T06:16:54,281 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T06:16:54,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-14T06:16:54,282 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T06:16:54,282 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T06:16:54,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:54,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43446 deadline: 1731565074357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:54,359 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:54,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:54,359 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:54,359 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:54,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43444 deadline: 1731565074357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:54,359 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:54,359 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:54,359 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:54,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-14T06:16:54,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-14T06:16:54,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:54,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. as already flushing 2024-11-14T06:16:54,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:54,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:54,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:54,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:54,488 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/2c603ec86ba342c7b2f4c1d5ff698ccf 2024-11-14T06:16:54,499 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/29e8f036925a4e1db44ab5dbf6740155 is 50, key is test_row_0/C:col10/1731565013158/Put/seqid=0 2024-11-14T06:16:54,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742022_1198 (size=12301) 2024-11-14T06:16:54,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-14T06:16:54,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:54,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. as already flushing 2024-11-14T06:16:54,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:54,596 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:54,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:54,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-14T06:16:54,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:54,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-14T06:16:54,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:54,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. as already flushing 2024-11-14T06:16:54,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:54,750 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:54,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:54,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:54,902 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-14T06:16:54,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:54,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. as already flushing 2024-11-14T06:16:54,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:54,903 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:54,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:54,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:54,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-14T06:16:54,940 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/29e8f036925a4e1db44ab5dbf6740155 2024-11-14T06:16:54,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/545a58322453491fab76a9bb5bc12c6f as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/545a58322453491fab76a9bb5bc12c6f 2024-11-14T06:16:54,959 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/545a58322453491fab76a9bb5bc12c6f, entries=200, sequenceid=291, filesize=39.0 K 2024-11-14T06:16:54,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/2c603ec86ba342c7b2f4c1d5ff698ccf as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/2c603ec86ba342c7b2f4c1d5ff698ccf 2024-11-14T06:16:54,967 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/2c603ec86ba342c7b2f4c1d5ff698ccf, entries=150, sequenceid=291, filesize=12.0 K 2024-11-14T06:16:54,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/29e8f036925a4e1db44ab5dbf6740155 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/29e8f036925a4e1db44ab5dbf6740155 2024-11-14T06:16:54,979 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/29e8f036925a4e1db44ab5dbf6740155, entries=150, sequenceid=291, filesize=12.0 K 2024-11-14T06:16:54,981 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 715443fb55c29fd4a00fd2e9afbe3938 in 1822ms, sequenceid=291, compaction requested=true 2024-11-14T06:16:54,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 715443fb55c29fd4a00fd2e9afbe3938: 2024-11-14T06:16:54,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 715443fb55c29fd4a00fd2e9afbe3938:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T06:16:54,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:16:54,981 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T06:16:54,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 715443fb55c29fd4a00fd2e9afbe3938:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T06:16:54,981 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T06:16:54,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:16:54,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 715443fb55c29fd4a00fd2e9afbe3938:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T06:16:54,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:16:54,983 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142536 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T06:16:54,983 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): 715443fb55c29fd4a00fd2e9afbe3938/A is initiating minor compaction (all files) 2024-11-14T06:16:54,983 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 715443fb55c29fd4a00fd2e9afbe3938/A in TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:54,984 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/4d5c5ba8aca640158244b85848068103, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/36a6414b86334c3e89e0b4415de8719b, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/12692f8cd5564659ae326bed7d000d87, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/545a58322453491fab76a9bb5bc12c6f] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp, totalSize=139.2 K 2024-11-14T06:16:54,984 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:54,984 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. files: [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/4d5c5ba8aca640158244b85848068103, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/36a6414b86334c3e89e0b4415de8719b, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/12692f8cd5564659ae326bed7d000d87, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/545a58322453491fab76a9bb5bc12c6f] 2024-11-14T06:16:54,986 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d5c5ba8aca640158244b85848068103, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731565006998 2024-11-14T06:16:54,986 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49382 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T06:16:54,986 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): 715443fb55c29fd4a00fd2e9afbe3938/B is initiating minor compaction (all files) 2024-11-14T06:16:54,986 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 715443fb55c29fd4a00fd2e9afbe3938/B in TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:54,986 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/378d6ffc43f64f44bf9633954e9b6618, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/04d1ca038df349ed80deba946ba42ece, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/acd7219ab0bd4bb98eadf3ce2cafe864, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/2c603ec86ba342c7b2f4c1d5ff698ccf] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp, totalSize=48.2 K 2024-11-14T06:16:54,987 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36a6414b86334c3e89e0b4415de8719b, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731565009149 2024-11-14T06:16:54,988 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 378d6ffc43f64f44bf9633954e9b6618, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731565006998 2024-11-14T06:16:54,988 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12692f8cd5564659ae326bed7d000d87, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731565010538 2024-11-14T06:16:54,989 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 04d1ca038df349ed80deba946ba42ece, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731565009149 2024-11-14T06:16:54,989 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 545a58322453491fab76a9bb5bc12c6f, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731565010956 2024-11-14T06:16:54,991 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting acd7219ab0bd4bb98eadf3ce2cafe864, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731565010538 2024-11-14T06:16:54,991 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c603ec86ba342c7b2f4c1d5ff698ccf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731565010956 2024-11-14T06:16:55,003 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 715443fb55c29fd4a00fd2e9afbe3938#B#compaction#165 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:16:55,004 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/77a1c60f5a334dbe8a062ddbddc73a7b is 50, key is test_row_0/B:col10/1731565013158/Put/seqid=0 2024-11-14T06:16:55,005 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=715443fb55c29fd4a00fd2e9afbe3938] 2024-11-14T06:16:55,037 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411149727fcc55d884c409211e07bfb2ff1d0_715443fb55c29fd4a00fd2e9afbe3938 store=[table=TestAcidGuarantees family=A region=715443fb55c29fd4a00fd2e9afbe3938] 2024-11-14T06:16:55,040 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411149727fcc55d884c409211e07bfb2ff1d0_715443fb55c29fd4a00fd2e9afbe3938, store=[table=TestAcidGuarantees family=A region=715443fb55c29fd4a00fd2e9afbe3938] 2024-11-14T06:16:55,040 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411149727fcc55d884c409211e07bfb2ff1d0_715443fb55c29fd4a00fd2e9afbe3938 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=715443fb55c29fd4a00fd2e9afbe3938] 2024-11-14T06:16:55,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-14T06:16:55,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:55,055 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 715443fb55c29fd4a00fd2e9afbe3938 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-14T06:16:55,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 715443fb55c29fd4a00fd2e9afbe3938, store=A 2024-11-14T06:16:55,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:55,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 715443fb55c29fd4a00fd2e9afbe3938, store=B 2024-11-14T06:16:55,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:55,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 715443fb55c29fd4a00fd2e9afbe3938, store=C 2024-11-14T06:16:55,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:55,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742023_1199 (size=12915) 2024-11-14T06:16:55,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742024_1200 (size=4469) 2024-11-14T06:16:55,069 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 715443fb55c29fd4a00fd2e9afbe3938#A#compaction#166 average throughput is 0.38 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:16:55,069 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/60b8ead2e01d474284b65ea14f91fc1d is 175, key is test_row_0/A:col10/1731565013158/Put/seqid=0 2024-11-14T06:16:55,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114f120072a2302437fa0cf20a1096398ac_715443fb55c29fd4a00fd2e9afbe3938 is 50, key is test_row_0/A:col10/1731565013204/Put/seqid=0 2024-11-14T06:16:55,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742025_1201 (size=31869) 2024-11-14T06:16:55,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742026_1202 (size=12454) 2024-11-14T06:16:55,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:55,111 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114f120072a2302437fa0cf20a1096398ac_715443fb55c29fd4a00fd2e9afbe3938 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f120072a2302437fa0cf20a1096398ac_715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:55,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/22ab4735139940f69bbeb0a78a9c2b09, store: [table=TestAcidGuarantees family=A region=715443fb55c29fd4a00fd2e9afbe3938] 2024-11-14T06:16:55,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/22ab4735139940f69bbeb0a78a9c2b09 is 175, key is test_row_0/A:col10/1731565013204/Put/seqid=0 2024-11-14T06:16:55,117 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/60b8ead2e01d474284b65ea14f91fc1d as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/60b8ead2e01d474284b65ea14f91fc1d 2024-11-14T06:16:55,135 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 715443fb55c29fd4a00fd2e9afbe3938/A of 715443fb55c29fd4a00fd2e9afbe3938 into 60b8ead2e01d474284b65ea14f91fc1d(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:16:55,135 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 715443fb55c29fd4a00fd2e9afbe3938: 2024-11-14T06:16:55,135 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., storeName=715443fb55c29fd4a00fd2e9afbe3938/A, priority=12, startTime=1731565014981; duration=0sec 2024-11-14T06:16:55,135 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:16:55,135 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 715443fb55c29fd4a00fd2e9afbe3938:A 2024-11-14T06:16:55,135 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T06:16:55,137 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49382 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T06:16:55,137 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): 715443fb55c29fd4a00fd2e9afbe3938/C is initiating minor compaction (all files) 2024-11-14T06:16:55,137 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 715443fb55c29fd4a00fd2e9afbe3938/C in TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:55,137 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/76bc5f78d39b4659bb535cac41635477, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/3b0a24b4ced14636a390de780dcd64f8, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/c75d2d2c006144b483a9de2fa6cee18d, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/29e8f036925a4e1db44ab5dbf6740155] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp, totalSize=48.2 K 2024-11-14T06:16:55,137 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76bc5f78d39b4659bb535cac41635477, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731565006998 2024-11-14T06:16:55,138 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b0a24b4ced14636a390de780dcd64f8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731565009149 2024-11-14T06:16:55,138 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting c75d2d2c006144b483a9de2fa6cee18d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731565010538 2024-11-14T06:16:55,138 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29e8f036925a4e1db44ab5dbf6740155, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731565010956 2024-11-14T06:16:55,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742027_1203 (size=31255) 2024-11-14T06:16:55,148 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=311, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/22ab4735139940f69bbeb0a78a9c2b09 2024-11-14T06:16:55,161 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 715443fb55c29fd4a00fd2e9afbe3938#C#compaction#168 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:16:55,162 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/a5e1e547f1e24735a2ff06cca3232389 is 50, key is test_row_0/C:col10/1731565013158/Put/seqid=0 2024-11-14T06:16:55,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/10790f8bf8fd4cf6ba2b23978f492364 is 50, key is test_row_0/B:col10/1731565013204/Put/seqid=0 2024-11-14T06:16:55,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742029_1205 (size=12301) 2024-11-14T06:16:55,226 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/10790f8bf8fd4cf6ba2b23978f492364 2024-11-14T06:16:55,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742028_1204 (size=12915) 2024-11-14T06:16:55,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/c2177a2e13364d58b906424c9d5a84d9 is 50, key is test_row_0/C:col10/1731565013204/Put/seqid=0 2024-11-14T06:16:55,268 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/a5e1e547f1e24735a2ff06cca3232389 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/a5e1e547f1e24735a2ff06cca3232389 2024-11-14T06:16:55,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. as already flushing 2024-11-14T06:16:55,279 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 715443fb55c29fd4a00fd2e9afbe3938/C of 715443fb55c29fd4a00fd2e9afbe3938 into a5e1e547f1e24735a2ff06cca3232389(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:16:55,279 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 715443fb55c29fd4a00fd2e9afbe3938: 2024-11-14T06:16:55,279 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., storeName=715443fb55c29fd4a00fd2e9afbe3938/C, priority=12, startTime=1731565014981; duration=0sec 2024-11-14T06:16:55,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on 715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:55,279 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:16:55,279 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 715443fb55c29fd4a00fd2e9afbe3938:C 2024-11-14T06:16:55,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742030_1206 (size=12301) 2024-11-14T06:16:55,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:55,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43476 deadline: 1731565075314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:55,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:55,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:55,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:55,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:55,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43484 deadline: 1731565075317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:55,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:55,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:55,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:55,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:55,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43470 deadline: 1731565075317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:55,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:55,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:55,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:55,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:55,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43444 deadline: 1731565075372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:55,374 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to ef93ef3a83c5,42283,1731564965945 for 'test_row_1' in 715443fb55c29fd4a00fd2e9afbe3938 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T06:16:55,375 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:55,375 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:55,375 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:55,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:55,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43446 deadline: 1731565075378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:55,385 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to ef93ef3a83c5,42283,1731564965945 for 'test_row_0' in 715443fb55c29fd4a00fd2e9afbe3938 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T06:16:55,386 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:55,386 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:55,386 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:55,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-14T06:16:55,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:55,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43484 deadline: 1731565075427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:55,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:55,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:55,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:55,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:55,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:55,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43476 deadline: 1731565075428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:55,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43470 deadline: 1731565075428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:55,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:55,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:55,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:55,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:55,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:55,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:55,468 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/77a1c60f5a334dbe8a062ddbddc73a7b as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/77a1c60f5a334dbe8a062ddbddc73a7b 2024-11-14T06:16:55,482 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 715443fb55c29fd4a00fd2e9afbe3938/B of 715443fb55c29fd4a00fd2e9afbe3938 into 77a1c60f5a334dbe8a062ddbddc73a7b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:16:55,482 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 715443fb55c29fd4a00fd2e9afbe3938: 2024-11-14T06:16:55,482 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., storeName=715443fb55c29fd4a00fd2e9afbe3938/B, priority=12, startTime=1731565014981; duration=0sec 2024-11-14T06:16:55,482 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:16:55,482 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 715443fb55c29fd4a00fd2e9afbe3938:B 2024-11-14T06:16:55,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:55,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43484 deadline: 1731565075638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:55,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:55,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:55,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:55,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:55,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43476 deadline: 1731565075652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:55,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:55,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43470 deadline: 1731565075647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:55,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:55,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:55,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:55,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:55,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:55,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:55,700 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/c2177a2e13364d58b906424c9d5a84d9 2024-11-14T06:16:55,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/22ab4735139940f69bbeb0a78a9c2b09 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/22ab4735139940f69bbeb0a78a9c2b09 2024-11-14T06:16:55,719 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/22ab4735139940f69bbeb0a78a9c2b09, entries=150, sequenceid=311, filesize=30.5 K 2024-11-14T06:16:55,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/10790f8bf8fd4cf6ba2b23978f492364 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/10790f8bf8fd4cf6ba2b23978f492364 2024-11-14T06:16:55,742 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/10790f8bf8fd4cf6ba2b23978f492364, entries=150, sequenceid=311, filesize=12.0 K 2024-11-14T06:16:55,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/c2177a2e13364d58b906424c9d5a84d9 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/c2177a2e13364d58b906424c9d5a84d9 2024-11-14T06:16:55,771 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/c2177a2e13364d58b906424c9d5a84d9, entries=150, sequenceid=311, filesize=12.0 K 2024-11-14T06:16:55,772 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 715443fb55c29fd4a00fd2e9afbe3938 in 717ms, sequenceid=311, compaction requested=false 2024-11-14T06:16:55,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 715443fb55c29fd4a00fd2e9afbe3938: 2024-11-14T06:16:55,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:55,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-14T06:16:55,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-14T06:16:55,783 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-14T06:16:55,783 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4930 sec 2024-11-14T06:16:55,787 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.5070 sec 2024-11-14T06:16:55,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on 715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:55,952 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 715443fb55c29fd4a00fd2e9afbe3938 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-14T06:16:55,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 715443fb55c29fd4a00fd2e9afbe3938, store=A 2024-11-14T06:16:55,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:55,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 715443fb55c29fd4a00fd2e9afbe3938, store=B 2024-11-14T06:16:55,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:55,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 715443fb55c29fd4a00fd2e9afbe3938, store=C 2024-11-14T06:16:55,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:55,973 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111411fbd3a6e6824b3f8966f238b024859a_715443fb55c29fd4a00fd2e9afbe3938 is 50, key is test_row_0/A:col10/1731565015304/Put/seqid=0 2024-11-14T06:16:56,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742031_1207 (size=14994) 2024-11-14T06:16:56,016 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:56,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:56,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43476 deadline: 1731565076011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:56,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:56,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:56,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:56,022 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111411fbd3a6e6824b3f8966f238b024859a_715443fb55c29fd4a00fd2e9afbe3938 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111411fbd3a6e6824b3f8966f238b024859a_715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:56,024 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/7c46d45c83c848279de26a7fb78aeced, store: [table=TestAcidGuarantees family=A region=715443fb55c29fd4a00fd2e9afbe3938] 2024-11-14T06:16:56,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/7c46d45c83c848279de26a7fb78aeced is 175, key is test_row_0/A:col10/1731565015304/Put/seqid=0 2024-11-14T06:16:56,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:56,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43484 deadline: 1731565076018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:56,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:56,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43470 deadline: 1731565076018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:56,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:56,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:56,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:56,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:56,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:56,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:56,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742032_1208 (size=39949) 2024-11-14T06:16:56,048 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=327, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/7c46d45c83c848279de26a7fb78aeced 2024-11-14T06:16:56,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/a97005215f324633b1aa3eb6c8efe094 is 50, key is test_row_0/B:col10/1731565015304/Put/seqid=0 2024-11-14T06:16:56,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742033_1209 (size=12301) 2024-11-14T06:16:56,098 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/a97005215f324633b1aa3eb6c8efe094 2024-11-14T06:16:56,108 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/f5655701eef9414cb390624f072eef3a is 50, key is test_row_0/C:col10/1731565015304/Put/seqid=0 2024-11-14T06:16:56,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742034_1210 (size=12301) 2024-11-14T06:16:56,126 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/f5655701eef9414cb390624f072eef3a 2024-11-14T06:16:56,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:56,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43476 deadline: 1731565076127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:56,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:56,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:56,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:56,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/7c46d45c83c848279de26a7fb78aeced as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/7c46d45c83c848279de26a7fb78aeced 2024-11-14T06:16:56,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:56,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43470 deadline: 1731565076137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:56,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:56,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:56,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:56,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:56,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43484 deadline: 1731565076137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:56,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:56,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:56,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:56,142 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/7c46d45c83c848279de26a7fb78aeced, entries=200, sequenceid=327, filesize=39.0 K 2024-11-14T06:16:56,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/a97005215f324633b1aa3eb6c8efe094 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/a97005215f324633b1aa3eb6c8efe094 2024-11-14T06:16:56,147 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/a97005215f324633b1aa3eb6c8efe094, entries=150, sequenceid=327, filesize=12.0 K 2024-11-14T06:16:56,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/f5655701eef9414cb390624f072eef3a as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/f5655701eef9414cb390624f072eef3a 2024-11-14T06:16:56,153 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/f5655701eef9414cb390624f072eef3a, entries=150, sequenceid=327, filesize=12.0 K 2024-11-14T06:16:56,154 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 715443fb55c29fd4a00fd2e9afbe3938 in 202ms, sequenceid=327, compaction requested=true 2024-11-14T06:16:56,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 715443fb55c29fd4a00fd2e9afbe3938: 2024-11-14T06:16:56,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 715443fb55c29fd4a00fd2e9afbe3938:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T06:16:56,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:16:56,155 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:16:56,155 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:16:56,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 715443fb55c29fd4a00fd2e9afbe3938:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T06:16:56,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:16:56,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 715443fb55c29fd4a00fd2e9afbe3938:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T06:16:56,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:16:56,156 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:16:56,156 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:16:56,156 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): 715443fb55c29fd4a00fd2e9afbe3938/A is initiating minor compaction (all files) 2024-11-14T06:16:56,156 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): 715443fb55c29fd4a00fd2e9afbe3938/B is initiating minor compaction (all files) 2024-11-14T06:16:56,156 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 715443fb55c29fd4a00fd2e9afbe3938/A in TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:56,156 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 715443fb55c29fd4a00fd2e9afbe3938/B in TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:56,156 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/60b8ead2e01d474284b65ea14f91fc1d, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/22ab4735139940f69bbeb0a78a9c2b09, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/7c46d45c83c848279de26a7fb78aeced] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp, totalSize=100.7 K 2024-11-14T06:16:56,156 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/77a1c60f5a334dbe8a062ddbddc73a7b, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/10790f8bf8fd4cf6ba2b23978f492364, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/a97005215f324633b1aa3eb6c8efe094] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp, totalSize=36.6 K 2024-11-14T06:16:56,156 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:56,156 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. files: [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/60b8ead2e01d474284b65ea14f91fc1d, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/22ab4735139940f69bbeb0a78a9c2b09, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/7c46d45c83c848279de26a7fb78aeced] 2024-11-14T06:16:56,156 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60b8ead2e01d474284b65ea14f91fc1d, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731565010956 2024-11-14T06:16:56,157 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 77a1c60f5a334dbe8a062ddbddc73a7b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731565010956 2024-11-14T06:16:56,157 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 10790f8bf8fd4cf6ba2b23978f492364, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731565013204 2024-11-14T06:16:56,158 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22ab4735139940f69bbeb0a78a9c2b09, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731565013204 2024-11-14T06:16:56,158 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting a97005215f324633b1aa3eb6c8efe094, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1731565015304 2024-11-14T06:16:56,158 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c46d45c83c848279de26a7fb78aeced, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1731565015304 2024-11-14T06:16:56,168 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 715443fb55c29fd4a00fd2e9afbe3938#B#compaction#174 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:16:56,169 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/fa048e96b87e4c3b8155445c8f5d3dce is 50, key is test_row_0/B:col10/1731565015304/Put/seqid=0 2024-11-14T06:16:56,173 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=715443fb55c29fd4a00fd2e9afbe3938] 2024-11-14T06:16:56,177 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411146596875725514b3188d88540ef9d82cc_715443fb55c29fd4a00fd2e9afbe3938 store=[table=TestAcidGuarantees family=A region=715443fb55c29fd4a00fd2e9afbe3938] 2024-11-14T06:16:56,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742035_1211 (size=13017) 2024-11-14T06:16:56,179 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411146596875725514b3188d88540ef9d82cc_715443fb55c29fd4a00fd2e9afbe3938, store=[table=TestAcidGuarantees family=A region=715443fb55c29fd4a00fd2e9afbe3938] 2024-11-14T06:16:56,180 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411146596875725514b3188d88540ef9d82cc_715443fb55c29fd4a00fd2e9afbe3938 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=715443fb55c29fd4a00fd2e9afbe3938] 2024-11-14T06:16:56,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742036_1212 (size=4469) 2024-11-14T06:16:56,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on 715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:56,343 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 715443fb55c29fd4a00fd2e9afbe3938 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-14T06:16:56,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 715443fb55c29fd4a00fd2e9afbe3938, store=A 2024-11-14T06:16:56,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:56,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 715443fb55c29fd4a00fd2e9afbe3938, store=B 2024-11-14T06:16:56,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:56,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 715443fb55c29fd4a00fd2e9afbe3938, store=C 2024-11-14T06:16:56,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:56,357 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411144b94873b4ce545ac973bc14bfb113dc7_715443fb55c29fd4a00fd2e9afbe3938 is 50, key is test_row_0/A:col10/1731565016341/Put/seqid=0 2024-11-14T06:16:56,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:56,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43476 deadline: 1731565076362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:56,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:56,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:56,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:56,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:56,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43484 deadline: 1731565076364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:56,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:56,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:56,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:56,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:56,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43470 deadline: 1731565076372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:56,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:56,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:56,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:56,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742037_1213 (size=14994) 2024-11-14T06:16:56,390 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:56,399 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411144b94873b4ce545ac973bc14bfb113dc7_715443fb55c29fd4a00fd2e9afbe3938 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144b94873b4ce545ac973bc14bfb113dc7_715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:56,401 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/8a2b7800700846d7bb48460f80fd15d5, store: [table=TestAcidGuarantees family=A region=715443fb55c29fd4a00fd2e9afbe3938] 2024-11-14T06:16:56,401 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/8a2b7800700846d7bb48460f80fd15d5 is 175, key is test_row_0/A:col10/1731565016341/Put/seqid=0 2024-11-14T06:16:56,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-14T06:16:56,428 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:16:56,428 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T06:16:56,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-14T06:16:56,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-14T06:16:56,431 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T06:16:56,432 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T06:16:56,432 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T06:16:56,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742038_1214 (size=39949) 2024-11-14T06:16:56,436 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=347, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/8a2b7800700846d7bb48460f80fd15d5 2024-11-14T06:16:56,466 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/b0fe7448d46f430cb3867e5fed5e3960 is 50, key is test_row_0/B:col10/1731565016341/Put/seqid=0 2024-11-14T06:16:56,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:56,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43476 deadline: 1731565076477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:56,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:56,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:56,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:56,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:56,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:56,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43470 deadline: 1731565076487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:56,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43484 deadline: 1731565076487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:56,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:56,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:56,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:56,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742039_1215 (size=12301) 2024-11-14T06:16:56,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:56,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:56,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:56,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-14T06:16:56,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-14T06:16:56,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:56,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. as already flushing 2024-11-14T06:16:56,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:56,587 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:56,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:56,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:56,609 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/fa048e96b87e4c3b8155445c8f5d3dce as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/fa048e96b87e4c3b8155445c8f5d3dce 2024-11-14T06:16:56,619 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 715443fb55c29fd4a00fd2e9afbe3938#A#compaction#175 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:16:56,619 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/cf68fbfc070f4e8d8281804f6ef2a6f4 is 175, key is test_row_0/A:col10/1731565015304/Put/seqid=0 2024-11-14T06:16:56,629 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 715443fb55c29fd4a00fd2e9afbe3938/B of 715443fb55c29fd4a00fd2e9afbe3938 into fa048e96b87e4c3b8155445c8f5d3dce(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:16:56,629 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 715443fb55c29fd4a00fd2e9afbe3938: 2024-11-14T06:16:56,629 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., storeName=715443fb55c29fd4a00fd2e9afbe3938/B, priority=13, startTime=1731565016155; duration=0sec 2024-11-14T06:16:56,629 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:16:56,629 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 715443fb55c29fd4a00fd2e9afbe3938:B 2024-11-14T06:16:56,629 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:16:56,631 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:16:56,631 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): 715443fb55c29fd4a00fd2e9afbe3938/C is initiating minor compaction (all files) 2024-11-14T06:16:56,632 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 715443fb55c29fd4a00fd2e9afbe3938/C in TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:56,632 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/a5e1e547f1e24735a2ff06cca3232389, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/c2177a2e13364d58b906424c9d5a84d9, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/f5655701eef9414cb390624f072eef3a] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp, totalSize=36.6 K 2024-11-14T06:16:56,632 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting a5e1e547f1e24735a2ff06cca3232389, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731565010956 2024-11-14T06:16:56,633 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting c2177a2e13364d58b906424c9d5a84d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731565013204 2024-11-14T06:16:56,633 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting f5655701eef9414cb390624f072eef3a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1731565015304 2024-11-14T06:16:56,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742040_1216 (size=31971) 2024-11-14T06:16:56,650 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 715443fb55c29fd4a00fd2e9afbe3938#C#compaction#178 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:16:56,651 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/0e54f5cd614148bca54cf0edc187c767 is 50, key is test_row_0/C:col10/1731565015304/Put/seqid=0 2024-11-14T06:16:56,658 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/cf68fbfc070f4e8d8281804f6ef2a6f4 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/cf68fbfc070f4e8d8281804f6ef2a6f4 2024-11-14T06:16:56,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742041_1217 (size=13017) 2024-11-14T06:16:56,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:56,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43476 deadline: 1731565076687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:56,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:56,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:56,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:56,692 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 715443fb55c29fd4a00fd2e9afbe3938/A of 715443fb55c29fd4a00fd2e9afbe3938 into cf68fbfc070f4e8d8281804f6ef2a6f4(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:16:56,692 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 715443fb55c29fd4a00fd2e9afbe3938: 2024-11-14T06:16:56,692 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., storeName=715443fb55c29fd4a00fd2e9afbe3938/A, priority=13, startTime=1731565016154; duration=0sec 2024-11-14T06:16:56,692 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:16:56,692 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 715443fb55c29fd4a00fd2e9afbe3938:A 2024-11-14T06:16:56,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:56,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43484 deadline: 1731565076697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:56,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:56,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:56,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:56,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:56,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43470 deadline: 1731565076707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:56,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:56,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:56,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:56,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-14T06:16:56,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:56,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. as already flushing 2024-11-14T06:16:56,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:56,742 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:56,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:56,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:56,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-14T06:16:56,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-14T06:16:56,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:56,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. as already flushing 2024-11-14T06:16:56,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:56,896 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:56,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:56,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:56,897 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/b0fe7448d46f430cb3867e5fed5e3960 2024-11-14T06:16:56,904 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/268b2e3f8bc64976aceb932fe13c0336 is 50, key is test_row_0/C:col10/1731565016341/Put/seqid=0 2024-11-14T06:16:56,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742042_1218 (size=12301) 2024-11-14T06:16:56,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:56,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43476 deadline: 1731565076997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:57,000 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:57,000 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:57,000 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:57,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:57,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43484 deadline: 1731565077007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:57,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:57,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:57,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:57,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:57,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43470 deadline: 1731565077018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:57,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:57,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:57,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:57,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-14T06:16:57,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:57,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. as already flushing 2024-11-14T06:16:57,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:57,049 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:57,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:57,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:57,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-14T06:16:57,065 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/0e54f5cd614148bca54cf0edc187c767 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/0e54f5cd614148bca54cf0edc187c767 2024-11-14T06:16:57,072 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 715443fb55c29fd4a00fd2e9afbe3938/C of 715443fb55c29fd4a00fd2e9afbe3938 into 0e54f5cd614148bca54cf0edc187c767(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:16:57,072 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 715443fb55c29fd4a00fd2e9afbe3938: 2024-11-14T06:16:57,072 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., storeName=715443fb55c29fd4a00fd2e9afbe3938/C, priority=13, startTime=1731565016155; duration=0sec 2024-11-14T06:16:57,072 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:16:57,072 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 715443fb55c29fd4a00fd2e9afbe3938:C 2024-11-14T06:16:57,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-14T06:16:57,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:57,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. as already flushing 2024-11-14T06:16:57,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:57,205 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:57,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:57,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:57,308 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/268b2e3f8bc64976aceb932fe13c0336 2024-11-14T06:16:57,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/8a2b7800700846d7bb48460f80fd15d5 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/8a2b7800700846d7bb48460f80fd15d5 2024-11-14T06:16:57,323 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/8a2b7800700846d7bb48460f80fd15d5, entries=200, sequenceid=347, filesize=39.0 K 2024-11-14T06:16:57,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/b0fe7448d46f430cb3867e5fed5e3960 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/b0fe7448d46f430cb3867e5fed5e3960 2024-11-14T06:16:57,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,336 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/b0fe7448d46f430cb3867e5fed5e3960, entries=150, sequenceid=347, filesize=12.0 K 2024-11-14T06:16:57,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/268b2e3f8bc64976aceb932fe13c0336 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/268b2e3f8bc64976aceb932fe13c0336 2024-11-14T06:16:57,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,346 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/268b2e3f8bc64976aceb932fe13c0336, entries=150, sequenceid=347, filesize=12.0 K 2024-11-14T06:16:57,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,348 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 715443fb55c29fd4a00fd2e9afbe3938 in 1004ms, sequenceid=347, compaction requested=false 2024-11-14T06:16:57,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 715443fb55c29fd4a00fd2e9afbe3938: 2024-11-14T06:16:57,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,358 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-14T06:16:57,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:57,358 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 715443fb55c29fd4a00fd2e9afbe3938 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-14T06:16:57,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 715443fb55c29fd4a00fd2e9afbe3938, store=A 2024-11-14T06:16:57,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:57,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 715443fb55c29fd4a00fd2e9afbe3938, store=B 2024-11-14T06:16:57,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:57,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 715443fb55c29fd4a00fd2e9afbe3938, store=C 2024-11-14T06:16:57,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:57,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114cf5bebcbf82b4b1ea849900141b7b959_715443fb55c29fd4a00fd2e9afbe3938 is 50, key is test_row_0/A:col10/1731565016362/Put/seqid=0 2024-11-14T06:16:57,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742043_1219 (size=9914) 2024-11-14T06:16:57,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on 715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:57,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. as already flushing 2024-11-14T06:16:57,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:57,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43446 deadline: 1731565077469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:57,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:57,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43444 deadline: 1731565077473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:57,475 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:57,475 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:57,475 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:57,477 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:57,477 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:57,477 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:57,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:57,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43476 deadline: 1731565077508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:57,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:57,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:57,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:57,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:57,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43484 deadline: 1731565077518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:57,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:57,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:57,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:57,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:57,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43470 deadline: 1731565077528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:57,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:57,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:57,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:57,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-14T06:16:57,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:57,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43444 deadline: 1731565077578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:57,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:57,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:57,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:57,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:57,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43446 deadline: 1731565077587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:57,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:57,590 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:57,590 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:57,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:57,780 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114cf5bebcbf82b4b1ea849900141b7b959_715443fb55c29fd4a00fd2e9afbe3938 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114cf5bebcbf82b4b1ea849900141b7b959_715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:57,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/4950e7562b6143b2819e129b1aefb610, store: [table=TestAcidGuarantees family=A region=715443fb55c29fd4a00fd2e9afbe3938] 2024-11-14T06:16:57,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/4950e7562b6143b2819e129b1aefb610 is 175, key is test_row_0/A:col10/1731565016362/Put/seqid=0 2024-11-14T06:16:57,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742044_1220 (size=22561) 2024-11-14T06:16:57,787 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=362, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/4950e7562b6143b2819e129b1aefb610 2024-11-14T06:16:57,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:57,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43444 deadline: 1731565077788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:57,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:57,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:57,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:57,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/62e1dec0e73e4bcaba66089c9270a250 is 50, key is test_row_0/B:col10/1731565016362/Put/seqid=0 2024-11-14T06:16:57,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:57,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43446 deadline: 1731565077797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:57,799 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:57,799 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:57,799 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:57,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742045_1221 (size=9857) 2024-11-14T06:16:57,809 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/62e1dec0e73e4bcaba66089c9270a250 2024-11-14T06:16:57,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/2ce3cea083284058a3193ef98d8a7ab9 is 50, key is test_row_0/C:col10/1731565016362/Put/seqid=0 2024-11-14T06:16:57,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742046_1222 (size=9857) 2024-11-14T06:16:58,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:58,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43444 deadline: 1731565078097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:58,100 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:58,100 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:58,100 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:58,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:58,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43446 deadline: 1731565078107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:58,108 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:58,108 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:58,108 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:58,225 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/2ce3cea083284058a3193ef98d8a7ab9 2024-11-14T06:16:58,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/4950e7562b6143b2819e129b1aefb610 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/4950e7562b6143b2819e129b1aefb610 2024-11-14T06:16:58,240 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/4950e7562b6143b2819e129b1aefb610, entries=100, sequenceid=362, filesize=22.0 K 2024-11-14T06:16:58,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/62e1dec0e73e4bcaba66089c9270a250 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/62e1dec0e73e4bcaba66089c9270a250 2024-11-14T06:16:58,247 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/62e1dec0e73e4bcaba66089c9270a250, entries=100, sequenceid=362, filesize=9.6 K 2024-11-14T06:16:58,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/2ce3cea083284058a3193ef98d8a7ab9 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/2ce3cea083284058a3193ef98d8a7ab9 2024-11-14T06:16:58,263 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/2ce3cea083284058a3193ef98d8a7ab9, entries=100, sequenceid=362, filesize=9.6 K 2024-11-14T06:16:58,265 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 715443fb55c29fd4a00fd2e9afbe3938 in 907ms, sequenceid=362, compaction requested=true 2024-11-14T06:16:58,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 715443fb55c29fd4a00fd2e9afbe3938: 2024-11-14T06:16:58,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:58,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-14T06:16:58,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-14T06:16:58,268 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-14T06:16:58,268 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8340 sec 2024-11-14T06:16:58,270 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.8400 sec 2024-11-14T06:16:58,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on 715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:58,532 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 715443fb55c29fd4a00fd2e9afbe3938 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-14T06:16:58,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 715443fb55c29fd4a00fd2e9afbe3938, store=A 2024-11-14T06:16:58,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:16:58,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 715443fb55c29fd4a00fd2e9afbe3938, store=B 2024-11-14T06:16:58,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:16:58,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 715443fb55c29fd4a00fd2e9afbe3938, store=C 2024-11-14T06:16:58,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:16:58,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:58,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43470 deadline: 1731565078552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:58,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:58,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:58,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43476 deadline: 1731565078552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:58,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:58,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:58,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:58,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43484 deadline: 1731565078554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:58,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:58,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:58,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:58,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:58,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:58,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:58,563 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114e3ee283c825e4205b844cc4d01130180_715443fb55c29fd4a00fd2e9afbe3938 is 50, key is test_row_0/A:col10/1731565017467/Put/seqid=0 2024-11-14T06:16:58,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-14T06:16:58,578 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:16:58,578 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T06:16:58,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-14T06:16:58,580 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T06:16:58,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-14T06:16:58,584 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T06:16:58,584 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T06:16:58,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742047_1223 (size=14994) 2024-11-14T06:16:58,607 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:16:58,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:58,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43444 deadline: 1731565078607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:58,608 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:58,608 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:58,608 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:58,615 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114e3ee283c825e4205b844cc4d01130180_715443fb55c29fd4a00fd2e9afbe3938 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e3ee283c825e4205b844cc4d01130180_715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:58,619 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/b01ed8621bcf4142af08b7cdb058cda4, store: [table=TestAcidGuarantees family=A region=715443fb55c29fd4a00fd2e9afbe3938] 2024-11-14T06:16:58,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/b01ed8621bcf4142af08b7cdb058cda4 is 175, key is test_row_0/A:col10/1731565017467/Put/seqid=0 2024-11-14T06:16:58,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:58,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43446 deadline: 1731565078618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:58,623 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:58,623 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:58,623 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:58,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:58,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43476 deadline: 1731565078657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:58,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:58,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:58,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:58,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:58,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43470 deadline: 1731565078657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:58,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:58,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43484 deadline: 1731565078658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:58,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:58,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:58,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:58,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:58,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:58,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:58,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742048_1224 (size=39949) 2024-11-14T06:16:58,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-14T06:16:58,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-14T06:16:58,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:58,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. as already flushing 2024-11-14T06:16:58,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:58,737 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:58,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:58,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:58,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:58,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43470 deadline: 1731565078868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:58,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:58,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:58,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:58,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:58,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43484 deadline: 1731565078868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:58,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:58,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43476 deadline: 1731565078868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:58,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:58,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:58,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:58,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:58,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:58,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:58,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-14T06:16:58,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:58,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. as already flushing 2024-11-14T06:16:58,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:58,891 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:58,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:58,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:58,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-14T06:16:59,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-14T06:16:59,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:59,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. as already flushing 2024-11-14T06:16:59,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:59,046 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:59,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:59,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:59,077 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=383, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/b01ed8621bcf4142af08b7cdb058cda4 2024-11-14T06:16:59,094 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/48a40b4bdc9e4ce69a6753b126eb3f6b is 50, key is test_row_0/B:col10/1731565017467/Put/seqid=0 2024-11-14T06:16:59,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742049_1225 (size=12301) 2024-11-14T06:16:59,107 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/48a40b4bdc9e4ce69a6753b126eb3f6b 2024-11-14T06:16:59,115 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/8b89ce50f6954819b45a28e98547f048 is 50, key is test_row_0/C:col10/1731565017467/Put/seqid=0 2024-11-14T06:16:59,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742050_1226 (size=12301) 2024-11-14T06:16:59,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:59,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43470 deadline: 1731565079177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:59,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:59,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:59,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:59,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:59,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43484 deadline: 1731565079177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:59,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:59,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:59,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:16:59,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:59,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43476 deadline: 1731565079177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:16:59,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:59,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:16:59,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:16:59,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-14T06:16:59,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:59,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. as already flushing 2024-11-14T06:16:59,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:59,200 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:59,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:59,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:59,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-14T06:16:59,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-14T06:16:59,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:59,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. as already flushing 2024-11-14T06:16:59,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:59,353 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:59,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:59,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:59,487 INFO [Thread-644 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-644. 2024-11-14T06:16:59,487 DEBUG [Thread-644 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T06:16:59,488 DEBUG [Thread-644 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:16:59,488 DEBUG [Thread-644 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:16:59,488 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:16:59,492 INFO [Thread-646 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-646. 2024-11-14T06:16:59,492 DEBUG [Thread-646 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T06:16:59,493 DEBUG [Thread-646 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:16:59,493 DEBUG [Thread-646 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:16:59,493 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:16:59,495 INFO [Thread-650 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-650. 2024-11-14T06:16:59,495 DEBUG [Thread-650 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T06:16:59,495 DEBUG [Thread-650 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:16:59,495 DEBUG [Thread-650 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:16:59,495 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:16:59,498 INFO [Thread-648 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-648. 2024-11-14T06:16:59,498 DEBUG [Thread-648 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T06:16:59,498 DEBUG [Thread-648 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:16:59,498 DEBUG [Thread-648 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:16:59,498 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:16:59,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-14T06:16:59,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:59,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. as already flushing 2024-11-14T06:16:59,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:59,506 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:59,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:59,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:59,524 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/8b89ce50f6954819b45a28e98547f048 2024-11-14T06:16:59,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/b01ed8621bcf4142af08b7cdb058cda4 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/b01ed8621bcf4142af08b7cdb058cda4 2024-11-14T06:16:59,533 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/b01ed8621bcf4142af08b7cdb058cda4, entries=200, sequenceid=383, filesize=39.0 K 2024-11-14T06:16:59,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/48a40b4bdc9e4ce69a6753b126eb3f6b as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/48a40b4bdc9e4ce69a6753b126eb3f6b 2024-11-14T06:16:59,537 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/48a40b4bdc9e4ce69a6753b126eb3f6b, entries=150, sequenceid=383, filesize=12.0 K 2024-11-14T06:16:59,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/8b89ce50f6954819b45a28e98547f048 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/8b89ce50f6954819b45a28e98547f048 2024-11-14T06:16:59,542 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/8b89ce50f6954819b45a28e98547f048, entries=150, sequenceid=383, filesize=12.0 K 2024-11-14T06:16:59,543 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=60.38 KB/61830 for 715443fb55c29fd4a00fd2e9afbe3938 in 1010ms, sequenceid=383, compaction requested=true 2024-11-14T06:16:59,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 715443fb55c29fd4a00fd2e9afbe3938: 2024-11-14T06:16:59,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 715443fb55c29fd4a00fd2e9afbe3938:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T06:16:59,543 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T06:16:59,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:16:59,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 715443fb55c29fd4a00fd2e9afbe3938:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T06:16:59,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:16:59,543 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T06:16:59,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 715443fb55c29fd4a00fd2e9afbe3938:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T06:16:59,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:16:59,544 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134430 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T06:16:59,544 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47476 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T06:16:59,544 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): 715443fb55c29fd4a00fd2e9afbe3938/A is initiating minor compaction (all files) 2024-11-14T06:16:59,544 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): 715443fb55c29fd4a00fd2e9afbe3938/B is initiating minor compaction (all files) 2024-11-14T06:16:59,544 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 715443fb55c29fd4a00fd2e9afbe3938/A in TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:59,544 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 715443fb55c29fd4a00fd2e9afbe3938/B in TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:59,544 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/cf68fbfc070f4e8d8281804f6ef2a6f4, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/8a2b7800700846d7bb48460f80fd15d5, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/4950e7562b6143b2819e129b1aefb610, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/b01ed8621bcf4142af08b7cdb058cda4] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp, totalSize=131.3 K 2024-11-14T06:16:59,544 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/fa048e96b87e4c3b8155445c8f5d3dce, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/b0fe7448d46f430cb3867e5fed5e3960, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/62e1dec0e73e4bcaba66089c9270a250, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/48a40b4bdc9e4ce69a6753b126eb3f6b] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp, totalSize=46.4 K 2024-11-14T06:16:59,544 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:59,544 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. files: [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/cf68fbfc070f4e8d8281804f6ef2a6f4, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/8a2b7800700846d7bb48460f80fd15d5, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/4950e7562b6143b2819e129b1aefb610, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/b01ed8621bcf4142af08b7cdb058cda4] 2024-11-14T06:16:59,544 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting fa048e96b87e4c3b8155445c8f5d3dce, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1731565015304 2024-11-14T06:16:59,545 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf68fbfc070f4e8d8281804f6ef2a6f4, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1731565015304 2024-11-14T06:16:59,545 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting b0fe7448d46f430cb3867e5fed5e3960, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731565016006 2024-11-14T06:16:59,545 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a2b7800700846d7bb48460f80fd15d5, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731565016006 2024-11-14T06:16:59,545 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 62e1dec0e73e4bcaba66089c9270a250, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1731565016361 2024-11-14T06:16:59,545 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4950e7562b6143b2819e129b1aefb610, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1731565016361 2024-11-14T06:16:59,545 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 48a40b4bdc9e4ce69a6753b126eb3f6b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1731565017467 2024-11-14T06:16:59,545 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting b01ed8621bcf4142af08b7cdb058cda4, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1731565017449 2024-11-14T06:16:59,555 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=715443fb55c29fd4a00fd2e9afbe3938] 2024-11-14T06:16:59,556 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 715443fb55c29fd4a00fd2e9afbe3938#B#compaction#186 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:16:59,556 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/7ac26f7c872a4caca6ff908cabba2bca is 50, key is test_row_0/B:col10/1731565017467/Put/seqid=0 2024-11-14T06:16:59,558 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411143b2dcca0e8f4497e86ce9ea899a55f52_715443fb55c29fd4a00fd2e9afbe3938 store=[table=TestAcidGuarantees family=A region=715443fb55c29fd4a00fd2e9afbe3938] 2024-11-14T06:16:59,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742051_1227 (size=13153) 2024-11-14T06:16:59,581 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411143b2dcca0e8f4497e86ce9ea899a55f52_715443fb55c29fd4a00fd2e9afbe3938, store=[table=TestAcidGuarantees family=A region=715443fb55c29fd4a00fd2e9afbe3938] 2024-11-14T06:16:59,581 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411143b2dcca0e8f4497e86ce9ea899a55f52_715443fb55c29fd4a00fd2e9afbe3938 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=715443fb55c29fd4a00fd2e9afbe3938] 2024-11-14T06:16:59,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742052_1228 (size=4469) 2024-11-14T06:16:59,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on 715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:16:59,619 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 715443fb55c29fd4a00fd2e9afbe3938 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-14T06:16:59,619 INFO [Thread-641 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-641. 2024-11-14T06:16:59,619 DEBUG [Thread-641 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T06:16:59,619 DEBUG [Thread-641 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:16:59,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 715443fb55c29fd4a00fd2e9afbe3938, store=A 2024-11-14T06:16:59,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:59,619 DEBUG [Thread-641 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:16:59,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 715443fb55c29fd4a00fd2e9afbe3938, store=B 2024-11-14T06:16:59,620 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:16:59,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:59,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 715443fb55c29fd4a00fd2e9afbe3938, store=C 2024-11-14T06:16:59,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:16:59,627 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411145c6df70e08344cf98b4e7a6505630c1f_715443fb55c29fd4a00fd2e9afbe3938 is 50, key is test_row_0/A:col10/1731565018536/Put/seqid=0 2024-11-14T06:16:59,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742053_1229 (size=12454) 2024-11-14T06:16:59,639 INFO [Thread-635 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-635. 2024-11-14T06:16:59,639 DEBUG [Thread-635 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T06:16:59,639 DEBUG [Thread-635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:16:59,639 DEBUG [Thread-635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:16:59,639 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:16:59,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-14T06:16:59,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:59,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. as already flushing 2024-11-14T06:16:59,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:59,659 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:59,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:59,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:59,689 INFO [Thread-639 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-639. 2024-11-14T06:16:59,689 INFO [Thread-633 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-633. 2024-11-14T06:16:59,689 DEBUG [Thread-639 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T06:16:59,689 DEBUG [Thread-633 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T06:16:59,689 DEBUG [Thread-639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:16:59,689 DEBUG [Thread-633 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:16:59,689 DEBUG [Thread-633 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:16:59,689 DEBUG [Thread-639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:16:59,689 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:16:59,689 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:16:59,699 INFO [Thread-637 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-637. 2024-11-14T06:16:59,699 DEBUG [Thread-637 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T06:16:59,699 DEBUG [Thread-637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:16:59,699 DEBUG [Thread-637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:16:59,699 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:16:59,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-14T06:16:59,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-14T06:16:59,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:59,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. as already flushing 2024-11-14T06:16:59,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:59,812 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:59,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:59,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:59,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-14T06:16:59,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:59,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. as already flushing 2024-11-14T06:16:59,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:59,966 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:59,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:59,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:16:59,976 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/7ac26f7c872a4caca6ff908cabba2bca as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/7ac26f7c872a4caca6ff908cabba2bca 2024-11-14T06:16:59,981 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 715443fb55c29fd4a00fd2e9afbe3938/B of 715443fb55c29fd4a00fd2e9afbe3938 into 7ac26f7c872a4caca6ff908cabba2bca(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:16:59,981 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 715443fb55c29fd4a00fd2e9afbe3938: 2024-11-14T06:16:59,981 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., storeName=715443fb55c29fd4a00fd2e9afbe3938/B, priority=12, startTime=1731565019543; duration=0sec 2024-11-14T06:16:59,981 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:16:59,981 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 715443fb55c29fd4a00fd2e9afbe3938:B 2024-11-14T06:16:59,981 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T06:16:59,982 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47476 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T06:16:59,982 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): 715443fb55c29fd4a00fd2e9afbe3938/C is initiating minor compaction (all files) 2024-11-14T06:16:59,982 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 715443fb55c29fd4a00fd2e9afbe3938/C in TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:16:59,982 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/0e54f5cd614148bca54cf0edc187c767, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/268b2e3f8bc64976aceb932fe13c0336, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/2ce3cea083284058a3193ef98d8a7ab9, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/8b89ce50f6954819b45a28e98547f048] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp, totalSize=46.4 K 2024-11-14T06:16:59,983 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e54f5cd614148bca54cf0edc187c767, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1731565015304 2024-11-14T06:16:59,983 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 268b2e3f8bc64976aceb932fe13c0336, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731565016006 2024-11-14T06:16:59,983 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ce3cea083284058a3193ef98d8a7ab9, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1731565016361 2024-11-14T06:16:59,983 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b89ce50f6954819b45a28e98547f048, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1731565017467 2024-11-14T06:16:59,986 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 715443fb55c29fd4a00fd2e9afbe3938#A#compaction#187 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:16:59,987 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/603db7be6ca648a69ba8828a4d411c97 is 175, key is test_row_0/A:col10/1731565017467/Put/seqid=0 2024-11-14T06:16:59,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742054_1230 (size=32107) 2024-11-14T06:16:59,997 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 715443fb55c29fd4a00fd2e9afbe3938#C#compaction#189 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:16:59,998 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/85e2e70121604508812111e14b0464c2 is 50, key is test_row_0/C:col10/1731565017467/Put/seqid=0 2024-11-14T06:17:00,004 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/603db7be6ca648a69ba8828a4d411c97 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/603db7be6ca648a69ba8828a4d411c97 2024-11-14T06:17:00,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742055_1231 (size=13153) 2024-11-14T06:17:00,016 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 715443fb55c29fd4a00fd2e9afbe3938/A of 715443fb55c29fd4a00fd2e9afbe3938 into 603db7be6ca648a69ba8828a4d411c97(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:17:00,016 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 715443fb55c29fd4a00fd2e9afbe3938: 2024-11-14T06:17:00,016 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., storeName=715443fb55c29fd4a00fd2e9afbe3938/A, priority=12, startTime=1731565019543; duration=0sec 2024-11-14T06:17:00,016 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:00,016 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 715443fb55c29fd4a00fd2e9afbe3938:A 2024-11-14T06:17:00,037 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:00,042 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411145c6df70e08344cf98b4e7a6505630c1f_715443fb55c29fd4a00fd2e9afbe3938 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145c6df70e08344cf98b4e7a6505630c1f_715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:17:00,043 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/bc70651e33bb4cce97be486647353c2b, store: [table=TestAcidGuarantees family=A region=715443fb55c29fd4a00fd2e9afbe3938] 2024-11-14T06:17:00,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/bc70651e33bb4cce97be486647353c2b is 175, key is test_row_0/A:col10/1731565018536/Put/seqid=0 2024-11-14T06:17:00,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742056_1232 (size=31255) 2024-11-14T06:17:00,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-14T06:17:00,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:17:00,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. as already flushing 2024-11-14T06:17:00,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:17:00,119 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:00,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:00,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:00,272 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-14T06:17:00,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:17:00,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. as already flushing 2024-11-14T06:17:00,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:17:00,273 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:00,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:00,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:00,417 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/85e2e70121604508812111e14b0464c2 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/85e2e70121604508812111e14b0464c2 2024-11-14T06:17:00,423 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 715443fb55c29fd4a00fd2e9afbe3938/C of 715443fb55c29fd4a00fd2e9afbe3938 into 85e2e70121604508812111e14b0464c2(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:17:00,423 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 715443fb55c29fd4a00fd2e9afbe3938: 2024-11-14T06:17:00,423 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938., storeName=715443fb55c29fd4a00fd2e9afbe3938/C, priority=12, startTime=1731565019543; duration=0sec 2024-11-14T06:17:00,423 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:00,423 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 715443fb55c29fd4a00fd2e9afbe3938:C 2024-11-14T06:17:00,447 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=396, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/bc70651e33bb4cce97be486647353c2b 2024-11-14T06:17:00,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/7ef1298a80bc44319dc197ba31a97282 is 50, key is test_row_0/B:col10/1731565018536/Put/seqid=0 2024-11-14T06:17:00,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-14T06:17:00,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:17:00,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. as already flushing 2024-11-14T06:17:00,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:17:00,464 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:00,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742057_1233 (size=12301) 2024-11-14T06:17:00,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:00,465 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/7ef1298a80bc44319dc197ba31a97282 2024-11-14T06:17:00,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:00,477 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/5223c481933748f083bee3f0dc05ec2c is 50, key is test_row_0/C:col10/1731565018536/Put/seqid=0 2024-11-14T06:17:00,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742058_1234 (size=12301) 2024-11-14T06:17:00,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-14T06:17:00,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:17:00,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. as already flushing 2024-11-14T06:17:00,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:17:00,618 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:00,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:00,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:00,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-14T06:17:00,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-14T06:17:00,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:17:00,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. as already flushing 2024-11-14T06:17:00,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:17:00,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:00,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:00,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:00,881 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/5223c481933748f083bee3f0dc05ec2c 2024-11-14T06:17:00,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/bc70651e33bb4cce97be486647353c2b as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/bc70651e33bb4cce97be486647353c2b 2024-11-14T06:17:00,890 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/bc70651e33bb4cce97be486647353c2b, entries=150, sequenceid=396, filesize=30.5 K 2024-11-14T06:17:00,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/7ef1298a80bc44319dc197ba31a97282 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/7ef1298a80bc44319dc197ba31a97282 2024-11-14T06:17:00,895 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/7ef1298a80bc44319dc197ba31a97282, entries=150, sequenceid=396, filesize=12.0 K 2024-11-14T06:17:00,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/5223c481933748f083bee3f0dc05ec2c as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/5223c481933748f083bee3f0dc05ec2c 2024-11-14T06:17:00,900 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/5223c481933748f083bee3f0dc05ec2c, entries=150, sequenceid=396, filesize=12.0 K 2024-11-14T06:17:00,901 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=26.84 KB/27480 for 715443fb55c29fd4a00fd2e9afbe3938 in 1281ms, sequenceid=396, compaction requested=false 2024-11-14T06:17:00,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 715443fb55c29fd4a00fd2e9afbe3938: 2024-11-14T06:17:00,923 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-14T06:17:00,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:17:00,924 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 715443fb55c29fd4a00fd2e9afbe3938 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-14T06:17:00,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 715443fb55c29fd4a00fd2e9afbe3938, store=A 2024-11-14T06:17:00,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:00,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 715443fb55c29fd4a00fd2e9afbe3938, store=B 2024-11-14T06:17:00,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:00,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 715443fb55c29fd4a00fd2e9afbe3938, store=C 2024-11-14T06:17:00,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:00,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114aa2d7091a42348f080db2ed2727a9230_715443fb55c29fd4a00fd2e9afbe3938 is 50, key is test_row_0/A:col10/1731565019697/Put/seqid=0 2024-11-14T06:17:00,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742059_1235 (size=9914) 2024-11-14T06:17:01,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:01,344 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114aa2d7091a42348f080db2ed2727a9230_715443fb55c29fd4a00fd2e9afbe3938 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114aa2d7091a42348f080db2ed2727a9230_715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:17:01,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/e36d7615bf404780bb0ade9759f0915d, store: [table=TestAcidGuarantees family=A region=715443fb55c29fd4a00fd2e9afbe3938] 2024-11-14T06:17:01,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/e36d7615bf404780bb0ade9759f0915d is 175, key is test_row_0/A:col10/1731565019697/Put/seqid=0 2024-11-14T06:17:01,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742060_1236 (size=22561) 2024-11-14T06:17:01,751 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=406, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/e36d7615bf404780bb0ade9759f0915d 2024-11-14T06:17:01,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/b3116af85bcc4cfab86076abdde4100e is 50, key is test_row_0/B:col10/1731565019697/Put/seqid=0 2024-11-14T06:17:01,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742061_1237 (size=9857) 2024-11-14T06:17:02,161 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/b3116af85bcc4cfab86076abdde4100e 2024-11-14T06:17:02,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/5a4fd545598549f3a75277fb8032ee77 is 50, key is test_row_0/C:col10/1731565019697/Put/seqid=0 2024-11-14T06:17:02,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742062_1238 (size=9857) 2024-11-14T06:17:02,574 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/5a4fd545598549f3a75277fb8032ee77 2024-11-14T06:17:02,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/A/e36d7615bf404780bb0ade9759f0915d as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/e36d7615bf404780bb0ade9759f0915d 2024-11-14T06:17:02,583 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/e36d7615bf404780bb0ade9759f0915d, entries=100, sequenceid=406, filesize=22.0 K 2024-11-14T06:17:02,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/B/b3116af85bcc4cfab86076abdde4100e as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/b3116af85bcc4cfab86076abdde4100e 2024-11-14T06:17:02,589 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/b3116af85bcc4cfab86076abdde4100e, entries=100, sequenceid=406, filesize=9.6 K 2024-11-14T06:17:02,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/.tmp/C/5a4fd545598549f3a75277fb8032ee77 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/5a4fd545598549f3a75277fb8032ee77 2024-11-14T06:17:02,594 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/5a4fd545598549f3a75277fb8032ee77, entries=100, sequenceid=406, filesize=9.6 K 2024-11-14T06:17:02,595 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 715443fb55c29fd4a00fd2e9afbe3938 in 1671ms, sequenceid=406, compaction requested=true 2024-11-14T06:17:02,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 715443fb55c29fd4a00fd2e9afbe3938: 2024-11-14T06:17:02,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:17:02,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-14T06:17:02,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-14T06:17:02,599 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-14T06:17:02,599 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.0130 sec 2024-11-14T06:17:02,600 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 4.0210 sec 2024-11-14T06:17:02,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-14T06:17:02,747 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:17:02,747 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T06:17:02,748 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-14T06:17:02,748 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 82 2024-11-14T06:17:02,748 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-11-14T06:17:02,748 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-11-14T06:17:02,748 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 75 2024-11-14T06:17:02,748 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T06:17:02,748 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1982 2024-11-14T06:17:02,748 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1960 2024-11-14T06:17:02,748 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T06:17:02,748 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 896 2024-11-14T06:17:02,748 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2688 rows 2024-11-14T06:17:02,748 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 885 2024-11-14T06:17:02,748 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2655 rows 2024-11-14T06:17:02,748 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T06:17:02,748 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T06:17:02,748 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:17:02,748 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:17:02,748 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:17:02,749 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-14T06:17:02,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T06:17:02,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-14T06:17:02,751 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731565022751"}]},"ts":"1731565022751"} 2024-11-14T06:17:02,753 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T06:17:02,753 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T06:17:02,753 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T06:17:02,755 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=715443fb55c29fd4a00fd2e9afbe3938, UNASSIGN}] 2024-11-14T06:17:02,756 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=715443fb55c29fd4a00fd2e9afbe3938, UNASSIGN 2024-11-14T06:17:02,756 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=715443fb55c29fd4a00fd2e9afbe3938, regionState=CLOSING, regionLocation=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:02,758 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=715443fb55c29fd4a00fd2e9afbe3938, UNASSIGN because future has completed 2024-11-14T06:17:02,758 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T06:17:02,758 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure 715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945}] 2024-11-14T06:17:02,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-14T06:17:02,911 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(122): Close 715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:17:02,911 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T06:17:02,911 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1722): Closing 715443fb55c29fd4a00fd2e9afbe3938, disabling compactions & flushes 2024-11-14T06:17:02,911 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:17:02,911 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:17:02,911 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. after waiting 0 ms 2024-11-14T06:17:02,911 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:17:02,912 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/a1186be964b14f66a3b84dfe4966754c, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/e080682261094c1ba373cbbc33efe5a2, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/3afd502297d24df58dfb5254939d0da1, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/3b72a3fab8e845bd93c373afd5535c3d, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/c3f337220a094bf58176e836fd60fa24, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/af394cc2cd3f465895eff9b7ec29a1a1, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/eccf45b0738b40419ff61477c4a08cc8, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/59a52cb27b844c28880ce9269b0f5179, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/4de44b33c21644e489e25dfccf3fdf9f, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/2ae0adcccfc84d2c8fb526f727a47b4f, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/fc3c0546dc37475c992790b9785b136f, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/826cc3fd68de438d8c2dbc0f7fb39040, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/3f84bb2ea6534781837a9d75c76f811c, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/07b1e010b3e84af389f12d64d5fd310e, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/4d5c5ba8aca640158244b85848068103, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/36a6414b86334c3e89e0b4415de8719b, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/12692f8cd5564659ae326bed7d000d87, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/545a58322453491fab76a9bb5bc12c6f, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/60b8ead2e01d474284b65ea14f91fc1d, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/22ab4735139940f69bbeb0a78a9c2b09, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/7c46d45c83c848279de26a7fb78aeced, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/cf68fbfc070f4e8d8281804f6ef2a6f4, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/8a2b7800700846d7bb48460f80fd15d5, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/4950e7562b6143b2819e129b1aefb610, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/b01ed8621bcf4142af08b7cdb058cda4] to archive 2024-11-14T06:17:02,913 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T06:17:02,914 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/a1186be964b14f66a3b84dfe4966754c to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/a1186be964b14f66a3b84dfe4966754c 2024-11-14T06:17:02,915 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/e080682261094c1ba373cbbc33efe5a2 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/e080682261094c1ba373cbbc33efe5a2 2024-11-14T06:17:02,916 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/3afd502297d24df58dfb5254939d0da1 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/3afd502297d24df58dfb5254939d0da1 2024-11-14T06:17:02,917 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/3b72a3fab8e845bd93c373afd5535c3d to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/3b72a3fab8e845bd93c373afd5535c3d 2024-11-14T06:17:02,918 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/c3f337220a094bf58176e836fd60fa24 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/c3f337220a094bf58176e836fd60fa24 2024-11-14T06:17:02,919 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/af394cc2cd3f465895eff9b7ec29a1a1 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/af394cc2cd3f465895eff9b7ec29a1a1 2024-11-14T06:17:02,920 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/eccf45b0738b40419ff61477c4a08cc8 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/eccf45b0738b40419ff61477c4a08cc8 2024-11-14T06:17:02,921 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/59a52cb27b844c28880ce9269b0f5179 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/59a52cb27b844c28880ce9269b0f5179 2024-11-14T06:17:02,922 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/4de44b33c21644e489e25dfccf3fdf9f to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/4de44b33c21644e489e25dfccf3fdf9f 2024-11-14T06:17:02,924 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/2ae0adcccfc84d2c8fb526f727a47b4f to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/2ae0adcccfc84d2c8fb526f727a47b4f 2024-11-14T06:17:02,925 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/fc3c0546dc37475c992790b9785b136f to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/fc3c0546dc37475c992790b9785b136f 2024-11-14T06:17:02,926 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/826cc3fd68de438d8c2dbc0f7fb39040 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/826cc3fd68de438d8c2dbc0f7fb39040 2024-11-14T06:17:02,927 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/3f84bb2ea6534781837a9d75c76f811c to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/3f84bb2ea6534781837a9d75c76f811c 2024-11-14T06:17:02,929 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/07b1e010b3e84af389f12d64d5fd310e to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/07b1e010b3e84af389f12d64d5fd310e 2024-11-14T06:17:02,930 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/4d5c5ba8aca640158244b85848068103 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/4d5c5ba8aca640158244b85848068103 2024-11-14T06:17:02,931 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/36a6414b86334c3e89e0b4415de8719b to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/36a6414b86334c3e89e0b4415de8719b 2024-11-14T06:17:02,932 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/12692f8cd5564659ae326bed7d000d87 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/12692f8cd5564659ae326bed7d000d87 2024-11-14T06:17:02,933 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/545a58322453491fab76a9bb5bc12c6f to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/545a58322453491fab76a9bb5bc12c6f 2024-11-14T06:17:02,934 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/60b8ead2e01d474284b65ea14f91fc1d to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/60b8ead2e01d474284b65ea14f91fc1d 2024-11-14T06:17:02,935 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/22ab4735139940f69bbeb0a78a9c2b09 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/22ab4735139940f69bbeb0a78a9c2b09 2024-11-14T06:17:02,936 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/7c46d45c83c848279de26a7fb78aeced to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/7c46d45c83c848279de26a7fb78aeced 2024-11-14T06:17:02,937 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/cf68fbfc070f4e8d8281804f6ef2a6f4 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/cf68fbfc070f4e8d8281804f6ef2a6f4 2024-11-14T06:17:02,938 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/8a2b7800700846d7bb48460f80fd15d5 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/8a2b7800700846d7bb48460f80fd15d5 2024-11-14T06:17:02,939 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/4950e7562b6143b2819e129b1aefb610 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/4950e7562b6143b2819e129b1aefb610 2024-11-14T06:17:02,940 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/b01ed8621bcf4142af08b7cdb058cda4 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/b01ed8621bcf4142af08b7cdb058cda4 2024-11-14T06:17:02,941 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/9af36e19d06342168b217f9b8d0bcad4, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/570f6541134d4cf99bfd4f820cfa9856, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/4ba1ef576f2c46c49dbc434640d141ee, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/871fea2b6b9946d3a8abcd94260e55b2, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/bbc27a327f054037a469722c424abc82, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/cdd582fbf177434b84f4ee173f9099d5, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/4bacb134f948428582d5e9b1d05c6e85, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/ff82186fe43445339060ad17cd8d170a, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/793a7e323d31487cb0299407353153cf, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/bc625f4f6a374c499b96ff5c4d96eb5e, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/a099012ff74b4710b32aee290970471a, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/3c20446a1bdd48999bf2ef2a58556c78, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/e52448332b5c4f8db9ba761778e2cb5a, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/378d6ffc43f64f44bf9633954e9b6618, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/03192e1cec07493a82d21beee2406d04, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/04d1ca038df349ed80deba946ba42ece, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/acd7219ab0bd4bb98eadf3ce2cafe864, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/77a1c60f5a334dbe8a062ddbddc73a7b, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/2c603ec86ba342c7b2f4c1d5ff698ccf, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/10790f8bf8fd4cf6ba2b23978f492364, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/fa048e96b87e4c3b8155445c8f5d3dce, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/a97005215f324633b1aa3eb6c8efe094, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/b0fe7448d46f430cb3867e5fed5e3960, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/62e1dec0e73e4bcaba66089c9270a250, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/48a40b4bdc9e4ce69a6753b126eb3f6b] to archive 2024-11-14T06:17:02,942 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T06:17:02,943 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/9af36e19d06342168b217f9b8d0bcad4 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/9af36e19d06342168b217f9b8d0bcad4 2024-11-14T06:17:02,944 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/570f6541134d4cf99bfd4f820cfa9856 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/570f6541134d4cf99bfd4f820cfa9856 2024-11-14T06:17:02,946 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/4ba1ef576f2c46c49dbc434640d141ee to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/4ba1ef576f2c46c49dbc434640d141ee 2024-11-14T06:17:02,947 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/871fea2b6b9946d3a8abcd94260e55b2 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/871fea2b6b9946d3a8abcd94260e55b2 2024-11-14T06:17:02,949 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/bbc27a327f054037a469722c424abc82 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/bbc27a327f054037a469722c424abc82 2024-11-14T06:17:02,950 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/cdd582fbf177434b84f4ee173f9099d5 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/cdd582fbf177434b84f4ee173f9099d5 2024-11-14T06:17:02,951 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/4bacb134f948428582d5e9b1d05c6e85 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/4bacb134f948428582d5e9b1d05c6e85 2024-11-14T06:17:02,952 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/ff82186fe43445339060ad17cd8d170a to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/ff82186fe43445339060ad17cd8d170a 2024-11-14T06:17:02,953 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/793a7e323d31487cb0299407353153cf to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/793a7e323d31487cb0299407353153cf 2024-11-14T06:17:02,954 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/bc625f4f6a374c499b96ff5c4d96eb5e to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/bc625f4f6a374c499b96ff5c4d96eb5e 2024-11-14T06:17:02,956 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/a099012ff74b4710b32aee290970471a to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/a099012ff74b4710b32aee290970471a 2024-11-14T06:17:02,957 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/3c20446a1bdd48999bf2ef2a58556c78 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/3c20446a1bdd48999bf2ef2a58556c78 2024-11-14T06:17:02,958 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/e52448332b5c4f8db9ba761778e2cb5a to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/e52448332b5c4f8db9ba761778e2cb5a 2024-11-14T06:17:02,959 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/378d6ffc43f64f44bf9633954e9b6618 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/378d6ffc43f64f44bf9633954e9b6618 2024-11-14T06:17:02,960 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/03192e1cec07493a82d21beee2406d04 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/03192e1cec07493a82d21beee2406d04 2024-11-14T06:17:02,961 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/04d1ca038df349ed80deba946ba42ece to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/04d1ca038df349ed80deba946ba42ece 2024-11-14T06:17:02,961 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/acd7219ab0bd4bb98eadf3ce2cafe864 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/acd7219ab0bd4bb98eadf3ce2cafe864 2024-11-14T06:17:02,962 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/77a1c60f5a334dbe8a062ddbddc73a7b to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/77a1c60f5a334dbe8a062ddbddc73a7b 2024-11-14T06:17:02,963 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/2c603ec86ba342c7b2f4c1d5ff698ccf to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/2c603ec86ba342c7b2f4c1d5ff698ccf 2024-11-14T06:17:02,965 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/10790f8bf8fd4cf6ba2b23978f492364 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/10790f8bf8fd4cf6ba2b23978f492364 2024-11-14T06:17:02,966 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/fa048e96b87e4c3b8155445c8f5d3dce to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/fa048e96b87e4c3b8155445c8f5d3dce 2024-11-14T06:17:02,967 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/a97005215f324633b1aa3eb6c8efe094 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/a97005215f324633b1aa3eb6c8efe094 2024-11-14T06:17:02,968 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/b0fe7448d46f430cb3867e5fed5e3960 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/b0fe7448d46f430cb3867e5fed5e3960 2024-11-14T06:17:02,969 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/62e1dec0e73e4bcaba66089c9270a250 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/62e1dec0e73e4bcaba66089c9270a250 2024-11-14T06:17:02,970 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/48a40b4bdc9e4ce69a6753b126eb3f6b to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/48a40b4bdc9e4ce69a6753b126eb3f6b 2024-11-14T06:17:02,972 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/612597fe630b42ce89c04d714f8754ff, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/158a1a8358b24f05abb46c68d13d6439, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/7d4fb4ed667749768523fe2e43285241, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/d1ec0ff934334b7fbf544a2f1a0be94d, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/e6c73f57204d4c0d9ae715d1c8120d55, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/cd956313276a486ba4fa604f322abce2, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/a7395012eb8d47cc8e9a0eb0271b8b94, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/1f44eb6b28294eec8c0f1c42a1ec061d, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/4d6bec6b538746739bcde75e94f0e448, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/468a5c60281540ab9df8098b39c535af, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/856d4b6837954c2289094c681ecb9859, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/1b6fbbae30e24052a5643719e54b5c2c, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/fc3023e3117d438f844458bd03e375d5, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/76bc5f78d39b4659bb535cac41635477, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/a2f133a05d6346e89ea0f93be09c6136, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/3b0a24b4ced14636a390de780dcd64f8, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/c75d2d2c006144b483a9de2fa6cee18d, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/a5e1e547f1e24735a2ff06cca3232389, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/29e8f036925a4e1db44ab5dbf6740155, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/c2177a2e13364d58b906424c9d5a84d9, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/0e54f5cd614148bca54cf0edc187c767, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/f5655701eef9414cb390624f072eef3a, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/268b2e3f8bc64976aceb932fe13c0336, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/2ce3cea083284058a3193ef98d8a7ab9, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/8b89ce50f6954819b45a28e98547f048] to archive 2024-11-14T06:17:02,973 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T06:17:02,974 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/612597fe630b42ce89c04d714f8754ff to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/612597fe630b42ce89c04d714f8754ff 2024-11-14T06:17:02,975 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/158a1a8358b24f05abb46c68d13d6439 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/158a1a8358b24f05abb46c68d13d6439 2024-11-14T06:17:02,976 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/7d4fb4ed667749768523fe2e43285241 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/7d4fb4ed667749768523fe2e43285241 2024-11-14T06:17:02,978 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/d1ec0ff934334b7fbf544a2f1a0be94d to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/d1ec0ff934334b7fbf544a2f1a0be94d 2024-11-14T06:17:02,979 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/e6c73f57204d4c0d9ae715d1c8120d55 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/e6c73f57204d4c0d9ae715d1c8120d55 2024-11-14T06:17:02,980 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/cd956313276a486ba4fa604f322abce2 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/cd956313276a486ba4fa604f322abce2 2024-11-14T06:17:02,982 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/a7395012eb8d47cc8e9a0eb0271b8b94 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/a7395012eb8d47cc8e9a0eb0271b8b94 2024-11-14T06:17:02,983 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/1f44eb6b28294eec8c0f1c42a1ec061d to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/1f44eb6b28294eec8c0f1c42a1ec061d 2024-11-14T06:17:02,985 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/4d6bec6b538746739bcde75e94f0e448 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/4d6bec6b538746739bcde75e94f0e448 2024-11-14T06:17:02,986 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/468a5c60281540ab9df8098b39c535af to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/468a5c60281540ab9df8098b39c535af 2024-11-14T06:17:02,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/856d4b6837954c2289094c681ecb9859 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/856d4b6837954c2289094c681ecb9859 2024-11-14T06:17:02,989 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/1b6fbbae30e24052a5643719e54b5c2c to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/1b6fbbae30e24052a5643719e54b5c2c 2024-11-14T06:17:02,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/fc3023e3117d438f844458bd03e375d5 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/fc3023e3117d438f844458bd03e375d5 2024-11-14T06:17:02,991 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/76bc5f78d39b4659bb535cac41635477 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/76bc5f78d39b4659bb535cac41635477 2024-11-14T06:17:02,992 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/a2f133a05d6346e89ea0f93be09c6136 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/a2f133a05d6346e89ea0f93be09c6136 2024-11-14T06:17:02,994 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/3b0a24b4ced14636a390de780dcd64f8 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/3b0a24b4ced14636a390de780dcd64f8 2024-11-14T06:17:02,995 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/c75d2d2c006144b483a9de2fa6cee18d to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/c75d2d2c006144b483a9de2fa6cee18d 2024-11-14T06:17:02,996 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/a5e1e547f1e24735a2ff06cca3232389 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/a5e1e547f1e24735a2ff06cca3232389 2024-11-14T06:17:02,997 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/29e8f036925a4e1db44ab5dbf6740155 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/29e8f036925a4e1db44ab5dbf6740155 2024-11-14T06:17:02,999 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/c2177a2e13364d58b906424c9d5a84d9 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/c2177a2e13364d58b906424c9d5a84d9 2024-11-14T06:17:03,000 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/0e54f5cd614148bca54cf0edc187c767 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/0e54f5cd614148bca54cf0edc187c767 2024-11-14T06:17:03,001 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/f5655701eef9414cb390624f072eef3a to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/f5655701eef9414cb390624f072eef3a 2024-11-14T06:17:03,002 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/268b2e3f8bc64976aceb932fe13c0336 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/268b2e3f8bc64976aceb932fe13c0336 2024-11-14T06:17:03,003 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/2ce3cea083284058a3193ef98d8a7ab9 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/2ce3cea083284058a3193ef98d8a7ab9 2024-11-14T06:17:03,004 DEBUG [StoreCloser-TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/8b89ce50f6954819b45a28e98547f048 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/8b89ce50f6954819b45a28e98547f048 2024-11-14T06:17:03,009 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/recovered.edits/409.seqid, newMaxSeqId=409, maxSeqId=4 2024-11-14T06:17:03,010 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938. 2024-11-14T06:17:03,010 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1676): Region close journal for 715443fb55c29fd4a00fd2e9afbe3938: Waiting for close lock at 1731565022911Running coprocessor pre-close hooks at 1731565022911Disabling compacts and flushes for region at 1731565022911Disabling writes for close at 1731565022911Writing region close event to WAL at 1731565023006 (+95 ms)Running coprocessor post-close hooks at 1731565023009 (+3 ms)Closed at 1731565023010 (+1 ms) 2024-11-14T06:17:03,012 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(157): Closed 715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:17:03,012 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=715443fb55c29fd4a00fd2e9afbe3938, regionState=CLOSED 2024-11-14T06:17:03,014 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure 715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 because future has completed 2024-11-14T06:17:03,016 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-14T06:17:03,016 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseRegionProcedure 715443fb55c29fd4a00fd2e9afbe3938, server=ef93ef3a83c5,42283,1731564965945 in 257 msec 2024-11-14T06:17:03,017 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=55 2024-11-14T06:17:03,017 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=55, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=715443fb55c29fd4a00fd2e9afbe3938, UNASSIGN in 261 msec 2024-11-14T06:17:03,019 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-14T06:17:03,019 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 265 msec 2024-11-14T06:17:03,020 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731565023020"}]},"ts":"1731565023020"} 2024-11-14T06:17:03,021 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T06:17:03,021 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T06:17:03,023 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 273 msec 2024-11-14T06:17:03,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-14T06:17:03,067 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:17:03,068 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-14T06:17:03,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T06:17:03,070 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T06:17:03,071 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=58, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T06:17:03,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-14T06:17:03,074 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:17:03,077 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A, FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B, FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C, FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/recovered.edits] 2024-11-14T06:17:03,079 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/603db7be6ca648a69ba8828a4d411c97 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/603db7be6ca648a69ba8828a4d411c97 2024-11-14T06:17:03,080 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/bc70651e33bb4cce97be486647353c2b to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/bc70651e33bb4cce97be486647353c2b 2024-11-14T06:17:03,082 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/e36d7615bf404780bb0ade9759f0915d to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/A/e36d7615bf404780bb0ade9759f0915d 2024-11-14T06:17:03,084 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/7ac26f7c872a4caca6ff908cabba2bca to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/7ac26f7c872a4caca6ff908cabba2bca 2024-11-14T06:17:03,085 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/7ef1298a80bc44319dc197ba31a97282 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/7ef1298a80bc44319dc197ba31a97282 2024-11-14T06:17:03,086 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/b3116af85bcc4cfab86076abdde4100e to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/B/b3116af85bcc4cfab86076abdde4100e 2024-11-14T06:17:03,089 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/5223c481933748f083bee3f0dc05ec2c to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/5223c481933748f083bee3f0dc05ec2c 2024-11-14T06:17:03,090 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/5a4fd545598549f3a75277fb8032ee77 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/5a4fd545598549f3a75277fb8032ee77 2024-11-14T06:17:03,091 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/85e2e70121604508812111e14b0464c2 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/C/85e2e70121604508812111e14b0464c2 2024-11-14T06:17:03,094 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/recovered.edits/409.seqid to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938/recovered.edits/409.seqid 2024-11-14T06:17:03,096 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:17:03,096 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T06:17:03,096 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T06:17:03,097 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-14T06:17:03,101 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411140148c56a7dd64fb987d6768c57c2fc6f_715443fb55c29fd4a00fd2e9afbe3938 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411140148c56a7dd64fb987d6768c57c2fc6f_715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:17:03,103 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111406327244ce5643b9b1e305223a3b97c0_715443fb55c29fd4a00fd2e9afbe3938 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111406327244ce5643b9b1e305223a3b97c0_715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:17:03,115 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111411fbd3a6e6824b3f8966f238b024859a_715443fb55c29fd4a00fd2e9afbe3938 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111411fbd3a6e6824b3f8966f238b024859a_715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:17:03,117 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411141433d3e589634021b628c96595e9fc41_715443fb55c29fd4a00fd2e9afbe3938 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411141433d3e589634021b628c96595e9fc41_715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:17:03,119 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111423a7cb7c8b99456498c2d6a6279ebbe7_715443fb55c29fd4a00fd2e9afbe3938 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111423a7cb7c8b99456498c2d6a6279ebbe7_715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:17:03,121 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111444837ad1420342d9ae77679297bdd30f_715443fb55c29fd4a00fd2e9afbe3938 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111444837ad1420342d9ae77679297bdd30f_715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:17:03,123 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144b94873b4ce545ac973bc14bfb113dc7_715443fb55c29fd4a00fd2e9afbe3938 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144b94873b4ce545ac973bc14bfb113dc7_715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:17:03,125 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145c6df70e08344cf98b4e7a6505630c1f_715443fb55c29fd4a00fd2e9afbe3938 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145c6df70e08344cf98b4e7a6505630c1f_715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:17:03,128 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411146d8840f859dd4d4cbcb4d02606381b4d_715443fb55c29fd4a00fd2e9afbe3938 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411146d8840f859dd4d4cbcb4d02606381b4d_715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:17:03,130 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411146f1f116575484bbfb23dd710d399b0d8_715443fb55c29fd4a00fd2e9afbe3938 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411146f1f116575484bbfb23dd710d399b0d8_715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:17:03,132 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411149aaa17081542465b9ef5e4c8cdfb0701_715443fb55c29fd4a00fd2e9afbe3938 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411149aaa17081542465b9ef5e4c8cdfb0701_715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:17:03,134 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411149cdea612b0ed4d68aa011a74c8246d8d_715443fb55c29fd4a00fd2e9afbe3938 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411149cdea612b0ed4d68aa011a74c8246d8d_715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:17:03,136 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114aa2d7091a42348f080db2ed2727a9230_715443fb55c29fd4a00fd2e9afbe3938 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114aa2d7091a42348f080db2ed2727a9230_715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:17:03,137 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c355ba56b38e45568166f824d467f428_715443fb55c29fd4a00fd2e9afbe3938 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c355ba56b38e45568166f824d467f428_715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:17:03,139 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114cddd3d0462a243f3a7179a9f2b3492dd_715443fb55c29fd4a00fd2e9afbe3938 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114cddd3d0462a243f3a7179a9f2b3492dd_715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:17:03,141 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114cf5bebcbf82b4b1ea849900141b7b959_715443fb55c29fd4a00fd2e9afbe3938 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114cf5bebcbf82b4b1ea849900141b7b959_715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:17:03,142 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d62ec2557e064134842ceafd4c6b6969_715443fb55c29fd4a00fd2e9afbe3938 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d62ec2557e064134842ceafd4c6b6969_715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:17:03,144 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114dcb2fd7a12af4c299e3b3eeffa525082_715443fb55c29fd4a00fd2e9afbe3938 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114dcb2fd7a12af4c299e3b3eeffa525082_715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:17:03,146 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e3ee283c825e4205b844cc4d01130180_715443fb55c29fd4a00fd2e9afbe3938 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e3ee283c825e4205b844cc4d01130180_715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:17:03,147 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f120072a2302437fa0cf20a1096398ac_715443fb55c29fd4a00fd2e9afbe3938 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f120072a2302437fa0cf20a1096398ac_715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:17:03,149 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f3d8ec31f2d648a4b2f5ca4e52e40fba_715443fb55c29fd4a00fd2e9afbe3938 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f3d8ec31f2d648a4b2f5ca4e52e40fba_715443fb55c29fd4a00fd2e9afbe3938 2024-11-14T06:17:03,150 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T06:17:03,154 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=58, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T06:17:03,157 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T06:17:03,160 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T06:17:03,162 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=58, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T06:17:03,162 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T06:17:03,162 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731565023162"}]},"ts":"9223372036854775807"} 2024-11-14T06:17:03,165 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T06:17:03,165 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 715443fb55c29fd4a00fd2e9afbe3938, NAME => 'TestAcidGuarantees,,1731564996185.715443fb55c29fd4a00fd2e9afbe3938.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T06:17:03,165 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T06:17:03,166 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731565023165"}]},"ts":"9223372036854775807"} 2024-11-14T06:17:03,169 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T06:17:03,170 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=58, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T06:17:03,172 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 102 msec 2024-11-14T06:17:03,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-14T06:17:03,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T06:17:03,178 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:17:03,190 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=255 (was 249) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-20636308_22 at /127.0.0.1:41728 [Waiting for operation #238] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_37528205_22 at /127.0.0.1:41710 [Waiting for operation #246] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_37528205_22 at /127.0.0.1:43780 [Waiting for operation #521] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5ae4a4a-2751-c4f4-7a28-da4c7cf058aa/cluster_7988e05c-75f7-7e7d-81fe-ae6ab708e25a/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5ae4a4a-2751-c4f4-7a28-da4c7cf058aa/cluster_7988e05c-75f7-7e7d-81fe-ae6ab708e25a/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-20636308_22 at /127.0.0.1:40076 [Waiting for operation #79] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=461 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=884 (was 668) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=547 (was 1898) 2024-11-14T06:17:03,201 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=255, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=884, ProcessCount=11, AvailableMemoryMB=544 2024-11-14T06:17:03,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T06:17:03,203 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T06:17:03,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T06:17:03,205 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T06:17:03,205 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:03,205 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 59 2024-11-14T06:17:03,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-14T06:17:03,206 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T06:17:03,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742063_1239 (size=1061) 2024-11-14T06:17:03,215 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02 2024-11-14T06:17:03,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742064_1240 (size=53) 2024-11-14T06:17:03,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-14T06:17:03,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-14T06:17:03,622 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T06:17:03,622 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 5be089a788a7af479f29a70b78ed76e8, disabling compactions & flushes 2024-11-14T06:17:03,622 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:03,622 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:03,622 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. after waiting 0 ms 2024-11-14T06:17:03,622 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:03,623 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:03,623 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5be089a788a7af479f29a70b78ed76e8: Waiting for close lock at 1731565023622Disabling compacts and flushes for region at 1731565023622Disabling writes for close at 1731565023622Writing region close event to WAL at 1731565023622Closed at 1731565023622 2024-11-14T06:17:03,624 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T06:17:03,624 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731565023624"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731565023624"}]},"ts":"1731565023624"} 2024-11-14T06:17:03,625 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T06:17:03,626 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T06:17:03,626 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731565023626"}]},"ts":"1731565023626"} 2024-11-14T06:17:03,628 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T06:17:03,628 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5be089a788a7af479f29a70b78ed76e8, ASSIGN}] 2024-11-14T06:17:03,629 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5be089a788a7af479f29a70b78ed76e8, ASSIGN 2024-11-14T06:17:03,630 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5be089a788a7af479f29a70b78ed76e8, ASSIGN; state=OFFLINE, location=ef93ef3a83c5,42283,1731564965945; forceNewPlan=false, retain=false 2024-11-14T06:17:03,781 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=5be089a788a7af479f29a70b78ed76e8, regionState=OPENING, regionLocation=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:03,782 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5be089a788a7af479f29a70b78ed76e8, ASSIGN because future has completed 2024-11-14T06:17:03,782 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945}] 2024-11-14T06:17:03,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-14T06:17:03,937 INFO [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:03,937 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} 2024-11-14T06:17:03,938 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5be089a788a7af479f29a70b78ed76e8 2024-11-14T06:17:03,938 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T06:17:03,938 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7794): checking encryption for 5be089a788a7af479f29a70b78ed76e8 2024-11-14T06:17:03,938 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7797): checking classloading for 5be089a788a7af479f29a70b78ed76e8 2024-11-14T06:17:03,940 INFO [StoreOpener-5be089a788a7af479f29a70b78ed76e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5be089a788a7af479f29a70b78ed76e8 2024-11-14T06:17:03,941 INFO [StoreOpener-5be089a788a7af479f29a70b78ed76e8-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T06:17:03,942 INFO [StoreOpener-5be089a788a7af479f29a70b78ed76e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5be089a788a7af479f29a70b78ed76e8 columnFamilyName A 2024-11-14T06:17:03,942 DEBUG [StoreOpener-5be089a788a7af479f29a70b78ed76e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:03,944 INFO [StoreOpener-5be089a788a7af479f29a70b78ed76e8-1 {}] regionserver.HStore(327): Store=5be089a788a7af479f29a70b78ed76e8/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T06:17:03,944 INFO [StoreOpener-5be089a788a7af479f29a70b78ed76e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5be089a788a7af479f29a70b78ed76e8 2024-11-14T06:17:03,947 INFO [StoreOpener-5be089a788a7af479f29a70b78ed76e8-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T06:17:03,947 INFO [StoreOpener-5be089a788a7af479f29a70b78ed76e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5be089a788a7af479f29a70b78ed76e8 columnFamilyName B 2024-11-14T06:17:03,947 DEBUG [StoreOpener-5be089a788a7af479f29a70b78ed76e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:03,948 INFO [StoreOpener-5be089a788a7af479f29a70b78ed76e8-1 {}] regionserver.HStore(327): Store=5be089a788a7af479f29a70b78ed76e8/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T06:17:03,949 INFO [StoreOpener-5be089a788a7af479f29a70b78ed76e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5be089a788a7af479f29a70b78ed76e8 2024-11-14T06:17:03,950 INFO [StoreOpener-5be089a788a7af479f29a70b78ed76e8-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T06:17:03,950 INFO [StoreOpener-5be089a788a7af479f29a70b78ed76e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5be089a788a7af479f29a70b78ed76e8 columnFamilyName C 2024-11-14T06:17:03,950 DEBUG [StoreOpener-5be089a788a7af479f29a70b78ed76e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:03,951 INFO [StoreOpener-5be089a788a7af479f29a70b78ed76e8-1 {}] regionserver.HStore(327): Store=5be089a788a7af479f29a70b78ed76e8/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T06:17:03,951 INFO [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:03,951 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1038): replaying wal for 5be089a788a7af479f29a70b78ed76e8 2024-11-14T06:17:03,952 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8 2024-11-14T06:17:03,952 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8 2024-11-14T06:17:03,953 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1048): stopping wal replay for 5be089a788a7af479f29a70b78ed76e8 2024-11-14T06:17:03,953 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for 5be089a788a7af479f29a70b78ed76e8 2024-11-14T06:17:03,954 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T06:17:03,956 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1093): writing seq id for 5be089a788a7af479f29a70b78ed76e8 2024-11-14T06:17:03,958 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T06:17:03,959 INFO [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1114): Opened 5be089a788a7af479f29a70b78ed76e8; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73755618, jitterRate=0.09904435276985168}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T06:17:03,959 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5be089a788a7af479f29a70b78ed76e8 2024-11-14T06:17:03,959 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1006): Region open journal for 5be089a788a7af479f29a70b78ed76e8: Running coprocessor pre-open hook at 1731565023938Writing region info on filesystem at 1731565023938Initializing all the Stores at 1731565023939 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731565023939Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731565023939Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731565023939Cleaning up temporary data from old regions at 1731565023953 (+14 ms)Running coprocessor post-open hooks at 1731565023959 (+6 ms)Region opened successfully at 1731565023959 2024-11-14T06:17:03,960 INFO [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., pid=61, masterSystemTime=1731565023934 2024-11-14T06:17:03,962 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:03,963 INFO [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:03,963 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=5be089a788a7af479f29a70b78ed76e8, regionState=OPEN, openSeqNum=2, regionLocation=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:03,965 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 because future has completed 2024-11-14T06:17:03,968 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-14T06:17:03,968 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 in 184 msec 2024-11-14T06:17:03,970 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-11-14T06:17:03,970 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5be089a788a7af479f29a70b78ed76e8, ASSIGN in 340 msec 2024-11-14T06:17:03,971 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T06:17:03,971 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731565023971"}]},"ts":"1731565023971"} 2024-11-14T06:17:03,972 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T06:17:03,974 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T06:17:03,980 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 772 msec 2024-11-14T06:17:04,283 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T06:17:04,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-14T06:17:04,348 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:17:04,349 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52ef9e59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:04,349 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:17:04,349 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:17:04,350 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:17:04,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:17:04,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:17:04,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@477d2fc1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:04,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:17:04,351 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:17:04,351 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:17:04,352 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53526, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:17:04,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d5467ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:04,352 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:17:04,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:17:04,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:17:04,354 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33096, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:17:04,355 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T06:17:04,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is ef93ef3a83c5,36045,1731564965170 2024-11-14T06:17:04,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4d9b3121 2024-11-14T06:17:04,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T06:17:04,357 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53532, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T06:17:04,358 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31898a93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:04,359 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:17:04,359 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:17:04,359 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:17:04,359 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:17:04,360 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:17:04,360 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f282b94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:04,360 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:17:04,360 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:17:04,360 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:17:04,361 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53546, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:17:04,362 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7db419f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:04,362 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f95838d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:04,362 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:17:04,363 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:17:04,363 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:17:04,363 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:17:04,364 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:17:04,364 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79726149, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:04,364 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:17:04,364 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:17:04,364 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:17:04,365 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53576, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:17:04,366 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@383d21a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:04,367 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a612003, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:04,367 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:17:04,367 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:17:04,368 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:17:04,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:17:04,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:17:04,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dc6e565, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:04,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:17:04,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:17:04,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:17:04,369 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53594, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:17:04,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44f8ac00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:04,370 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@461a8f29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:04,371 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:17:04,371 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:17:04,372 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:17:04,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:17:04,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:17:04,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13eac863, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:04,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:17:04,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:17:04,373 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:17:04,373 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53614, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:17:04,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@655b6ccf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:04,375 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d37f322, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:04,375 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:17:04,375 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:17:04,376 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:17:04,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:17:04,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:17:04,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a4c684a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:04,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:17:04,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:17:04,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:17:04,377 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53638, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:17:04,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e64594d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:04,379 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19a99c3a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:04,379 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:17:04,379 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:17:04,380 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:17:04,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:17:04,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:17:04,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@226f88e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:04,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:17:04,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:17:04,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:17:04,381 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53668, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:17:04,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cb1ff4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:04,383 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28795458, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:04,383 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:17:04,383 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:17:04,384 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:17:04,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:17:04,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:17:04,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1233dd84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:04,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:17:04,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:17:04,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:17:04,385 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53686, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:17:04,386 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52693ba9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:04,387 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@532ccbbd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:04,387 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:17:04,387 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:17:04,388 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:17:04,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:17:04,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:17:04,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42283f50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:04,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:17:04,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:17:04,389 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:17:04,389 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53706, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:17:04,390 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@205bcdcc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:04,392 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4335b2cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:04,392 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:17:04,393 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:17:04,393 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:17:04,393 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:17:04,394 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:17:04,394 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18bda15b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:04,394 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:17:04,394 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:17:04,394 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:17:04,395 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53720, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:17:04,396 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a357a23, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:04,396 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@631e37b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:04,396 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:17:04,397 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:17:04,397 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:17:04,398 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:17:04,398 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:17:04,398 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ef5e5f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:04,398 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:17:04,398 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:17:04,398 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:17:04,399 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53732, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:17:04,400 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2eca21e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:04,425 DEBUG [Thread-1118 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:17:04,425 DEBUG [Thread-1133 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:17:04,425 DEBUG [Thread-1124 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:17:04,425 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T06:17:04,425 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:17:04,425 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:17:04,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:17:04,426 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:17:04,426 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:17:04,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:17:04,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-14T06:17:04,427 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33102, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:17:04,427 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33098, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:17:04,427 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33100, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:17:04,427 DEBUG [Thread-1120 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:17:04,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-14T06:17:04,428 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T06:17:04,428 DEBUG [Thread-1131 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:17:04,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:17:04,429 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:17:04,429 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T06:17:04,429 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T06:17:04,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:17:04,429 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:17:04,431 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33124, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:17:04,431 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33126, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:17:04,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2] 2024-11-14T06:17:04,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2] 2024-11-14T06:17:04,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2] 2024-11-14T06:17:04,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2] 2024-11-14T06:17:04,432 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2] 2024-11-14T06:17:04,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on 5be089a788a7af479f29a70b78ed76e8 2024-11-14T06:17:04,442 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5be089a788a7af479f29a70b78ed76e8 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-14T06:17:04,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=A 2024-11-14T06:17:04,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:04,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=B 2024-11-14T06:17:04,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:04,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=C 2024-11-14T06:17:04,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:04,448 DEBUG [Thread-1137 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:17:04,450 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:17:04,450 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:17:04,451 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33156, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:17:04,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2] 2024-11-14T06:17:04,456 DEBUG [Thread-1122 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:17:04,457 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:17:04,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:17:04,459 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33168, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:17:04,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2] 2024-11-14T06:17:04,471 DEBUG [Thread-1135 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:17:04,480 DEBUG [Thread-1135 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:17:04,480 DEBUG [Thread-1135 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:17:04,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:04,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33168 deadline: 1731565084475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:04,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:04,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:04,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:04,482 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33176, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:17:04,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:04,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33126 deadline: 1731565084477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:04,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:04,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:04,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:04,483 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/c55ef1e911e44107b5163fd380f6e409 is 50, key is test_row_0/A:col10/1731565024436/Put/seqid=0 2024-11-14T06:17:04,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2] 2024-11-14T06:17:04,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:04,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33100 deadline: 1731565084475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:04,484 DEBUG [Thread-1129 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:17:04,484 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:04,484 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:04,484 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:04,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:04,485 DEBUG [Thread-1129 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:17:04,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33098 deadline: 1731565084480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:04,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:04,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:04,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:04,490 DEBUG [Thread-1126 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:17:04,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:17:04,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:17:04,501 DEBUG [Thread-1129 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:17:04,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742065_1241 (size=12001) 2024-11-14T06:17:04,522 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/c55ef1e911e44107b5163fd380f6e409 2024-11-14T06:17:04,536 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33182, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:17:04,537 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33188, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:17:04,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-14T06:17:04,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2] 2024-11-14T06:17:04,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2] 2024-11-14T06:17:04,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:04,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 1 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33182 deadline: 1731565084543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:04,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:04,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:04,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:04,576 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/8028a263b8e647a39aaf38ba9a5dafb8 is 50, key is test_row_0/B:col10/1731565024436/Put/seqid=0 2024-11-14T06:17:04,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-14T06:17:04,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:04,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:04,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:04,587 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:04,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:04,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:04,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:04,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33126 deadline: 1731565084593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:04,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:04,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:04,600 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:04,600 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:04,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33100 deadline: 1731565084593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:04,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:04,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33098 deadline: 1731565084593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:04,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:04,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:04,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:04,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:04,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:04,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:04,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:04,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33168 deadline: 1731565084592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:04,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:04,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:04,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:04,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742066_1242 (size=12001) 2024-11-14T06:17:04,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:04,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 2 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33182 deadline: 1731565084657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:04,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:04,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:04,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:04,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-14T06:17:04,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:04,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:04,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:04,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:04,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:04,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:04,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-14T06:17:04,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:04,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33126 deadline: 1731565084808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:04,813 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:04,813 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:04,813 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:04,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:04,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33100 deadline: 1731565084809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:04,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:04,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:04,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:04,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:04,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33098 deadline: 1731565084809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:04,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:04,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:04,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:04,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:04,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33168 deadline: 1731565084809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:04,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:04,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:04,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:04,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:04,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33182 deadline: 1731565084867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:04,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:04,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:04,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:04,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-14T06:17:04,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:04,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:04,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:04,895 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:04,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:04,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:05,046 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/8028a263b8e647a39aaf38ba9a5dafb8 2024-11-14T06:17:05,051 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-14T06:17:05,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:05,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:05,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:05,052 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:05,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:05,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:05,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-14T06:17:05,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/331f73d6b28f46e8a19c5b0bbc0f7c91 is 50, key is test_row_0/C:col10/1731565024436/Put/seqid=0 2024-11-14T06:17:05,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:05,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33100 deadline: 1731565085118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:05,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:05,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:05,120 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:05,120 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:05,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33126 deadline: 1731565085118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:05,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:05,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:05,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:05,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:05,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33098 deadline: 1731565085127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:05,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:05,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33168 deadline: 1731565085127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:05,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:05,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:05,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:05,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:05,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:05,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:05,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742067_1243 (size=12001) 2024-11-14T06:17:05,161 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/331f73d6b28f46e8a19c5b0bbc0f7c91 2024-11-14T06:17:05,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/c55ef1e911e44107b5163fd380f6e409 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/c55ef1e911e44107b5163fd380f6e409 2024-11-14T06:17:05,183 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/c55ef1e911e44107b5163fd380f6e409, entries=150, sequenceid=13, filesize=11.7 K 2024-11-14T06:17:05,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:05,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33182 deadline: 1731565085177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:05,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:05,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:05,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:05,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/8028a263b8e647a39aaf38ba9a5dafb8 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/8028a263b8e647a39aaf38ba9a5dafb8 2024-11-14T06:17:05,199 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/8028a263b8e647a39aaf38ba9a5dafb8, entries=150, sequenceid=13, filesize=11.7 K 2024-11-14T06:17:05,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/331f73d6b28f46e8a19c5b0bbc0f7c91 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/331f73d6b28f46e8a19c5b0bbc0f7c91 2024-11-14T06:17:05,205 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-14T06:17:05,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:05,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:05,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:05,205 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:05,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:05,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:05,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/331f73d6b28f46e8a19c5b0bbc0f7c91, entries=150, sequenceid=13, filesize=11.7 K 2024-11-14T06:17:05,218 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 5be089a788a7af479f29a70b78ed76e8 in 776ms, sequenceid=13, compaction requested=false 2024-11-14T06:17:05,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5be089a788a7af479f29a70b78ed76e8: 2024-11-14T06:17:05,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-14T06:17:05,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:05,365 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 5be089a788a7af479f29a70b78ed76e8 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-14T06:17:05,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=A 2024-11-14T06:17:05,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:05,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=B 2024-11-14T06:17:05,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:05,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=C 2024-11-14T06:17:05,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:05,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/ac2a2d616f044090be12f8137efa9799 is 50, key is test_row_0/A:col10/1731565024473/Put/seqid=0 2024-11-14T06:17:05,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742068_1244 (size=12001) 2024-11-14T06:17:05,433 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/ac2a2d616f044090be12f8137efa9799 2024-11-14T06:17:05,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/cab1dcd54d924eb190a7f2a38f1061dc is 50, key is test_row_0/B:col10/1731565024473/Put/seqid=0 2024-11-14T06:17:05,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742069_1245 (size=12001) 2024-11-14T06:17:05,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-14T06:17:05,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:05,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on 5be089a788a7af479f29a70b78ed76e8 2024-11-14T06:17:05,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:05,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33168 deadline: 1731565085660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:05,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:05,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:05,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:05,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33126 deadline: 1731565085661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:05,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:05,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:05,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:05,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:05,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:05,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33098 deadline: 1731565085665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:05,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:05,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:05,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:05,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:05,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33100 deadline: 1731565085665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:05,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:05,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:05,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:05,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:05,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33182 deadline: 1731565085698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:05,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:05,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:05,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:05,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:05,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33126 deadline: 1731565085778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:05,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:05,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33098 deadline: 1731565085779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:05,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:05,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:05,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:05,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:05,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:05,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:05,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:05,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33100 deadline: 1731565085779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:05,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:05,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:05,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:05,901 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/cab1dcd54d924eb190a7f2a38f1061dc 2024-11-14T06:17:05,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/c078d0bc9bdc4f8c83a3e0ef57dbccc0 is 50, key is test_row_0/C:col10/1731565024473/Put/seqid=0 2024-11-14T06:17:05,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742070_1246 (size=12001) 2024-11-14T06:17:05,957 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/c078d0bc9bdc4f8c83a3e0ef57dbccc0 2024-11-14T06:17:05,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/ac2a2d616f044090be12f8137efa9799 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/ac2a2d616f044090be12f8137efa9799 2024-11-14T06:17:05,979 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/ac2a2d616f044090be12f8137efa9799, entries=150, sequenceid=33, filesize=11.7 K 2024-11-14T06:17:05,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/cab1dcd54d924eb190a7f2a38f1061dc as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/cab1dcd54d924eb190a7f2a38f1061dc 2024-11-14T06:17:05,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:05,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33098 deadline: 1731565085987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:05,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:05,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:05,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:05,989 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/cab1dcd54d924eb190a7f2a38f1061dc, entries=150, sequenceid=33, filesize=11.7 K 2024-11-14T06:17:05,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:05,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:05,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33100 deadline: 1731565085987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:05,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33126 deadline: 1731565085987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:05,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:05,990 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:05,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:05,990 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:05,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:05,990 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:05,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/c078d0bc9bdc4f8c83a3e0ef57dbccc0 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/c078d0bc9bdc4f8c83a3e0ef57dbccc0 2024-11-14T06:17:05,998 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/c078d0bc9bdc4f8c83a3e0ef57dbccc0, entries=150, sequenceid=33, filesize=11.7 K 2024-11-14T06:17:06,009 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 5be089a788a7af479f29a70b78ed76e8 in 644ms, sequenceid=33, compaction requested=false 2024-11-14T06:17:06,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 5be089a788a7af479f29a70b78ed76e8: 2024-11-14T06:17:06,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:06,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-14T06:17:06,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-14T06:17:06,018 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-14T06:17:06,018 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5830 sec 2024-11-14T06:17:06,023 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 1.5940 sec 2024-11-14T06:17:06,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on 5be089a788a7af479f29a70b78ed76e8 2024-11-14T06:17:06,301 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5be089a788a7af479f29a70b78ed76e8 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-14T06:17:06,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=A 2024-11-14T06:17:06,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:06,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=B 2024-11-14T06:17:06,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:06,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=C 2024-11-14T06:17:06,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:06,319 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/6a4d9fb2b0e64578aa4ba7258dc40415 is 50, key is test_row_0/A:col10/1731565025631/Put/seqid=0 2024-11-14T06:17:06,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742071_1247 (size=14341) 2024-11-14T06:17:06,356 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/6a4d9fb2b0e64578aa4ba7258dc40415 2024-11-14T06:17:06,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:06,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33100 deadline: 1731565086360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:06,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/bff334d5404a4512bdfaeae853b1f642 is 50, key is test_row_0/B:col10/1731565025631/Put/seqid=0 2024-11-14T06:17:06,374 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:06,374 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:06,374 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:06,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:06,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33126 deadline: 1731565086363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:06,382 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:06,382 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:06,382 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:06,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:06,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33098 deadline: 1731565086375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:06,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:06,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:06,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:06,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742072_1248 (size=12001) 2024-11-14T06:17:06,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:06,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33100 deadline: 1731565086478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:06,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:06,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:06,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:06,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:06,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33126 deadline: 1731565086487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:06,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:06,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:06,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:06,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:06,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33098 deadline: 1731565086500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:06,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:06,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:06,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:06,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-14T06:17:06,587 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:17:06,588 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T06:17:06,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-14T06:17:06,592 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T06:17:06,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-14T06:17:06,593 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T06:17:06,593 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T06:17:06,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:06,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33168 deadline: 1731565086677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:06,681 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to ef93ef3a83c5,42283,1731564965945 for 'test_row_1' in 5be089a788a7af479f29a70b78ed76e8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T06:17:06,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:06,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:06,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:06,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:06,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33100 deadline: 1731565086687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:06,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:06,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:06,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:06,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:06,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33126 deadline: 1731565086698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:06,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:06,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:06,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:06,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-14T06:17:06,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:06,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33182 deadline: 1731565086709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:06,714 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to ef93ef3a83c5,42283,1731564965945 for 'test_row_1' in 5be089a788a7af479f29a70b78ed76e8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T06:17:06,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:06,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:06,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:06,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:06,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33098 deadline: 1731565086717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:06,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:06,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:06,724 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:06,752 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-14T06:17:06,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:06,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:06,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:06,753 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:06,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:06,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:06,831 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/bff334d5404a4512bdfaeae853b1f642 2024-11-14T06:17:06,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/7290ab97d49f44b0a713d472a613ac23 is 50, key is test_row_0/C:col10/1731565025631/Put/seqid=0 2024-11-14T06:17:06,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742073_1249 (size=12001) 2024-11-14T06:17:06,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-14T06:17:06,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:06,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:06,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:06,907 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:06,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:06,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:06,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-14T06:17:07,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:07,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33100 deadline: 1731565086997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:07,001 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:07,001 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:07,001 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:07,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:07,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33126 deadline: 1731565087008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:07,013 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:07,013 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:07,013 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:07,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:07,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33098 deadline: 1731565087027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:07,028 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:07,028 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:07,028 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:07,061 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-14T06:17:07,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:07,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:07,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:07,061 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:07,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:07,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:07,216 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-14T06:17:07,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:07,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:07,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:07,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:07,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:07,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:07,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-14T06:17:07,302 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/7290ab97d49f44b0a713d472a613ac23 2024-11-14T06:17:07,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/6a4d9fb2b0e64578aa4ba7258dc40415 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/6a4d9fb2b0e64578aa4ba7258dc40415 2024-11-14T06:17:07,315 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/6a4d9fb2b0e64578aa4ba7258dc40415, entries=200, sequenceid=47, filesize=14.0 K 2024-11-14T06:17:07,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/bff334d5404a4512bdfaeae853b1f642 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/bff334d5404a4512bdfaeae853b1f642 2024-11-14T06:17:07,327 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/bff334d5404a4512bdfaeae853b1f642, entries=150, sequenceid=47, filesize=11.7 K 2024-11-14T06:17:07,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/7290ab97d49f44b0a713d472a613ac23 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/7290ab97d49f44b0a713d472a613ac23 2024-11-14T06:17:07,338 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/7290ab97d49f44b0a713d472a613ac23, entries=150, sequenceid=47, filesize=11.7 K 2024-11-14T06:17:07,339 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 5be089a788a7af479f29a70b78ed76e8 in 1038ms, sequenceid=47, compaction requested=true 2024-11-14T06:17:07,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5be089a788a7af479f29a70b78ed76e8: 2024-11-14T06:17:07,340 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:17:07,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5be089a788a7af479f29a70b78ed76e8:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T06:17:07,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:07,340 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:17:07,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5be089a788a7af479f29a70b78ed76e8:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T06:17:07,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:07,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5be089a788a7af479f29a70b78ed76e8:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T06:17:07,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:17:07,347 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:17:07,347 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): 5be089a788a7af479f29a70b78ed76e8/B is initiating minor compaction (all files) 2024-11-14T06:17:07,347 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5be089a788a7af479f29a70b78ed76e8/B in TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:07,347 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/8028a263b8e647a39aaf38ba9a5dafb8, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/cab1dcd54d924eb190a7f2a38f1061dc, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/bff334d5404a4512bdfaeae853b1f642] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp, totalSize=35.2 K 2024-11-14T06:17:07,347 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:17:07,347 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): 5be089a788a7af479f29a70b78ed76e8/A is initiating minor compaction (all files) 2024-11-14T06:17:07,348 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5be089a788a7af479f29a70b78ed76e8/A in TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:07,348 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/c55ef1e911e44107b5163fd380f6e409, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/ac2a2d616f044090be12f8137efa9799, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/6a4d9fb2b0e64578aa4ba7258dc40415] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp, totalSize=37.4 K 2024-11-14T06:17:07,348 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8028a263b8e647a39aaf38ba9a5dafb8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731565024436 2024-11-14T06:17:07,348 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting c55ef1e911e44107b5163fd380f6e409, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731565024436 2024-11-14T06:17:07,348 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting cab1dcd54d924eb190a7f2a38f1061dc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731565024473 2024-11-14T06:17:07,349 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting bff334d5404a4512bdfaeae853b1f642, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731565025631 2024-11-14T06:17:07,349 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting ac2a2d616f044090be12f8137efa9799, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731565024473 2024-11-14T06:17:07,349 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a4d9fb2b0e64578aa4ba7258dc40415, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731565025631 2024-11-14T06:17:07,371 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-14T06:17:07,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:07,371 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 5be089a788a7af479f29a70b78ed76e8 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-14T06:17:07,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=A 2024-11-14T06:17:07,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:07,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=B 2024-11-14T06:17:07,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:07,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=C 2024-11-14T06:17:07,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:07,374 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5be089a788a7af479f29a70b78ed76e8#B#compaction#204 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:17:07,375 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/31842e873f1240398dd798a62db9aac3 is 50, key is test_row_0/B:col10/1731565025631/Put/seqid=0 2024-11-14T06:17:07,379 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5be089a788a7af479f29a70b78ed76e8#A#compaction#206 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:17:07,379 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/071e6ee61edd4d819ee1d58a82a06124 is 50, key is test_row_0/A:col10/1731565025631/Put/seqid=0 2024-11-14T06:17:07,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/43794ab4db7d4f1c88635b8004d9d3ce is 50, key is test_row_0/A:col10/1731565026357/Put/seqid=0 2024-11-14T06:17:07,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742074_1250 (size=12104) 2024-11-14T06:17:07,463 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/31842e873f1240398dd798a62db9aac3 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/31842e873f1240398dd798a62db9aac3 2024-11-14T06:17:07,472 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5be089a788a7af479f29a70b78ed76e8/B of 5be089a788a7af479f29a70b78ed76e8 into 31842e873f1240398dd798a62db9aac3(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:17:07,472 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5be089a788a7af479f29a70b78ed76e8: 2024-11-14T06:17:07,472 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., storeName=5be089a788a7af479f29a70b78ed76e8/B, priority=13, startTime=1731565027340; duration=0sec 2024-11-14T06:17:07,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742076_1252 (size=12001) 2024-11-14T06:17:07,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742075_1251 (size=12104) 2024-11-14T06:17:07,474 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:17:07,474 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5be089a788a7af479f29a70b78ed76e8:B 2024-11-14T06:17:07,474 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:17:07,475 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/43794ab4db7d4f1c88635b8004d9d3ce 2024-11-14T06:17:07,476 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:17:07,477 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): 5be089a788a7af479f29a70b78ed76e8/C is initiating minor compaction (all files) 2024-11-14T06:17:07,478 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5be089a788a7af479f29a70b78ed76e8/C in TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:07,478 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/331f73d6b28f46e8a19c5b0bbc0f7c91, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/c078d0bc9bdc4f8c83a3e0ef57dbccc0, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/7290ab97d49f44b0a713d472a613ac23] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp, totalSize=35.2 K 2024-11-14T06:17:07,479 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 331f73d6b28f46e8a19c5b0bbc0f7c91, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731565024436 2024-11-14T06:17:07,481 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting c078d0bc9bdc4f8c83a3e0ef57dbccc0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731565024473 2024-11-14T06:17:07,482 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7290ab97d49f44b0a713d472a613ac23, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731565025631 2024-11-14T06:17:07,485 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/071e6ee61edd4d819ee1d58a82a06124 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/071e6ee61edd4d819ee1d58a82a06124 2024-11-14T06:17:07,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/4a0c0d4ef65646f4b18d8cfae6e8dfc6 is 50, key is test_row_0/B:col10/1731565026357/Put/seqid=0 2024-11-14T06:17:07,497 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5be089a788a7af479f29a70b78ed76e8/A of 5be089a788a7af479f29a70b78ed76e8 into 071e6ee61edd4d819ee1d58a82a06124(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:17:07,497 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5be089a788a7af479f29a70b78ed76e8: 2024-11-14T06:17:07,497 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., storeName=5be089a788a7af479f29a70b78ed76e8/A, priority=13, startTime=1731565027339; duration=0sec 2024-11-14T06:17:07,497 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:07,497 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5be089a788a7af479f29a70b78ed76e8:A 2024-11-14T06:17:07,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:07,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on 5be089a788a7af479f29a70b78ed76e8 2024-11-14T06:17:07,525 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5be089a788a7af479f29a70b78ed76e8#C#compaction#208 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:17:07,526 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/3ed337af46fb4a4db836ed00b65c470d is 50, key is test_row_0/C:col10/1731565025631/Put/seqid=0 2024-11-14T06:17:07,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:07,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33098 deadline: 1731565087554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:07,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:07,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:07,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:07,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:07,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33100 deadline: 1731565087557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:07,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:07,564 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:07,564 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:07,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33126 deadline: 1731565087558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:07,565 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:07,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:07,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:07,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:07,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742077_1253 (size=12001) 2024-11-14T06:17:07,571 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/4a0c0d4ef65646f4b18d8cfae6e8dfc6 2024-11-14T06:17:07,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742078_1254 (size=12104) 2024-11-14T06:17:07,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/96d322b77c6f4fa9b35173df55d73d58 is 50, key is test_row_0/C:col10/1731565026357/Put/seqid=0 2024-11-14T06:17:07,601 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/3ed337af46fb4a4db836ed00b65c470d as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/3ed337af46fb4a4db836ed00b65c470d 2024-11-14T06:17:07,609 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5be089a788a7af479f29a70b78ed76e8/C of 5be089a788a7af479f29a70b78ed76e8 into 3ed337af46fb4a4db836ed00b65c470d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:17:07,609 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5be089a788a7af479f29a70b78ed76e8: 2024-11-14T06:17:07,609 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., storeName=5be089a788a7af479f29a70b78ed76e8/C, priority=13, startTime=1731565027340; duration=0sec 2024-11-14T06:17:07,609 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:07,609 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5be089a788a7af479f29a70b78ed76e8:C 2024-11-14T06:17:07,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742079_1255 (size=12001) 2024-11-14T06:17:07,628 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/96d322b77c6f4fa9b35173df55d73d58 2024-11-14T06:17:07,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/43794ab4db7d4f1c88635b8004d9d3ce as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/43794ab4db7d4f1c88635b8004d9d3ce 2024-11-14T06:17:07,646 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/43794ab4db7d4f1c88635b8004d9d3ce, entries=150, sequenceid=66, filesize=11.7 K 2024-11-14T06:17:07,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/4a0c0d4ef65646f4b18d8cfae6e8dfc6 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/4a0c0d4ef65646f4b18d8cfae6e8dfc6 2024-11-14T06:17:07,654 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/4a0c0d4ef65646f4b18d8cfae6e8dfc6, entries=150, sequenceid=66, filesize=11.7 K 2024-11-14T06:17:07,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/96d322b77c6f4fa9b35173df55d73d58 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/96d322b77c6f4fa9b35173df55d73d58 2024-11-14T06:17:07,667 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/96d322b77c6f4fa9b35173df55d73d58, entries=150, sequenceid=66, filesize=11.7 K 2024-11-14T06:17:07,671 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 5be089a788a7af479f29a70b78ed76e8 in 299ms, sequenceid=66, compaction requested=false 2024-11-14T06:17:07,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 5be089a788a7af479f29a70b78ed76e8: 2024-11-14T06:17:07,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:07,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-14T06:17:07,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-14T06:17:07,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on 5be089a788a7af479f29a70b78ed76e8 2024-11-14T06:17:07,673 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5be089a788a7af479f29a70b78ed76e8 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-14T06:17:07,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=A 2024-11-14T06:17:07,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:07,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=B 2024-11-14T06:17:07,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:07,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=C 2024-11-14T06:17:07,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:07,675 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-14T06:17:07,675 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0790 sec 2024-11-14T06:17:07,679 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/9e43727d93334bd592b7913b3ebffc69 is 50, key is test_row_0/A:col10/1731565027556/Put/seqid=0 2024-11-14T06:17:07,679 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.0900 sec 2024-11-14T06:17:07,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-14T06:17:07,737 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:17:07,738 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T06:17:07,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-14T06:17:07,740 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T06:17:07,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-14T06:17:07,741 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T06:17:07,742 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T06:17:07,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742080_1256 (size=14341) 2024-11-14T06:17:07,756 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/9e43727d93334bd592b7913b3ebffc69 2024-11-14T06:17:07,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/04602e51209749a494a271df11ef8ecb is 50, key is test_row_0/B:col10/1731565027556/Put/seqid=0 2024-11-14T06:17:07,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:07,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33098 deadline: 1731565087781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:07,792 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:07,792 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:07,792 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:07,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:07,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33100 deadline: 1731565087791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:07,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:07,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33126 deadline: 1731565087794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:07,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:07,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:07,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:07,803 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:07,803 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:07,803 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:07,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742081_1257 (size=12001) 2024-11-14T06:17:07,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-14T06:17:07,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-14T06:17:07,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:07,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:07,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:07,895 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:07,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:07,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:07,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:07,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33098 deadline: 1731565087897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:07,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:07,908 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:07,908 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:07,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:07,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33100 deadline: 1731565087907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:07,911 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:07,911 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:07,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:07,911 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:07,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33126 deadline: 1731565087907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:07,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:07,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:07,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:08,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-14T06:17:08,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:08,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:08,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:08,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:08,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:08,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:08,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-14T06:17:08,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:08,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33100 deadline: 1731565088117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:08,122 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:08,122 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:08,122 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:08,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:08,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33126 deadline: 1731565088117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:08,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:08,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:08,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:08,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:08,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33098 deadline: 1731565088118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:08,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:08,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:08,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:08,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-14T06:17:08,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:08,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:08,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:08,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:08,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:08,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:08,225 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/04602e51209749a494a271df11ef8ecb 2024-11-14T06:17:08,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/85c1415983534c988b44b310f347c205 is 50, key is test_row_0/C:col10/1731565027556/Put/seqid=0 2024-11-14T06:17:08,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742082_1258 (size=12001) 2024-11-14T06:17:08,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-14T06:17:08,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-14T06:17:08,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:08,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:08,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:08,369 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:08,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:08,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:08,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:08,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33100 deadline: 1731565088427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:08,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:08,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:08,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33126 deadline: 1731565088427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:08,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33098 deadline: 1731565088427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:08,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:08,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:08,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:08,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:08,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:08,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:08,432 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:08,432 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:08,432 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:08,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-14T06:17:08,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:08,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:08,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:08,526 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:08,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:08,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:08,585 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T06:17:08,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-14T06:17:08,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:08,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:08,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:08,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:08,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:08,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:08,684 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/85c1415983534c988b44b310f347c205 2024-11-14T06:17:08,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/9e43727d93334bd592b7913b3ebffc69 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/9e43727d93334bd592b7913b3ebffc69 2024-11-14T06:17:08,694 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/9e43727d93334bd592b7913b3ebffc69, entries=200, sequenceid=83, filesize=14.0 K 2024-11-14T06:17:08,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/04602e51209749a494a271df11ef8ecb as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/04602e51209749a494a271df11ef8ecb 2024-11-14T06:17:08,703 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/04602e51209749a494a271df11ef8ecb, entries=150, sequenceid=83, filesize=11.7 K 2024-11-14T06:17:08,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/85c1415983534c988b44b310f347c205 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/85c1415983534c988b44b310f347c205 2024-11-14T06:17:08,714 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/85c1415983534c988b44b310f347c205, entries=150, sequenceid=83, filesize=11.7 K 2024-11-14T06:17:08,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:08,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33168 deadline: 1731565088710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:08,714 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to ef93ef3a83c5,42283,1731564965945 for 'test_row_1' in 5be089a788a7af479f29a70b78ed76e8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4239 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T06:17:08,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:08,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:08,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:08,716 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 5be089a788a7af479f29a70b78ed76e8 in 1043ms, sequenceid=83, compaction requested=true 2024-11-14T06:17:08,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5be089a788a7af479f29a70b78ed76e8: 2024-11-14T06:17:08,716 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:17:08,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5be089a788a7af479f29a70b78ed76e8:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T06:17:08,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:08,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5be089a788a7af479f29a70b78ed76e8:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T06:17:08,718 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:17:08,718 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): 5be089a788a7af479f29a70b78ed76e8/A is initiating minor compaction (all files) 2024-11-14T06:17:08,718 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5be089a788a7af479f29a70b78ed76e8/A in TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:08,718 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:17:08,718 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/071e6ee61edd4d819ee1d58a82a06124, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/43794ab4db7d4f1c88635b8004d9d3ce, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/9e43727d93334bd592b7913b3ebffc69] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp, totalSize=37.5 K 2024-11-14T06:17:08,718 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 071e6ee61edd4d819ee1d58a82a06124, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731565025631 2024-11-14T06:17:08,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:17:08,719 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43794ab4db7d4f1c88635b8004d9d3ce, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731565026331 2024-11-14T06:17:08,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5be089a788a7af479f29a70b78ed76e8:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T06:17:08,719 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e43727d93334bd592b7913b3ebffc69, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731565027544 2024-11-14T06:17:08,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:17:08,723 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:17:08,723 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): 5be089a788a7af479f29a70b78ed76e8/B is initiating minor compaction (all files) 2024-11-14T06:17:08,723 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5be089a788a7af479f29a70b78ed76e8/B in TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:08,724 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/31842e873f1240398dd798a62db9aac3, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/4a0c0d4ef65646f4b18d8cfae6e8dfc6, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/04602e51209749a494a271df11ef8ecb] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp, totalSize=35.3 K 2024-11-14T06:17:08,724 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 31842e873f1240398dd798a62db9aac3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731565025631 2024-11-14T06:17:08,726 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a0c0d4ef65646f4b18d8cfae6e8dfc6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731565026331 2024-11-14T06:17:08,726 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 04602e51209749a494a271df11ef8ecb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731565027544 2024-11-14T06:17:08,735 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5be089a788a7af479f29a70b78ed76e8#A#compaction#213 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:17:08,735 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/3bc774442fce4104adfd59e48d4cab74 is 50, key is test_row_0/A:col10/1731565027556/Put/seqid=0 2024-11-14T06:17:08,736 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5be089a788a7af479f29a70b78ed76e8 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-14T06:17:08,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=A 2024-11-14T06:17:08,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:08,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=B 2024-11-14T06:17:08,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:08,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=C 2024-11-14T06:17:08,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:08,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on 5be089a788a7af479f29a70b78ed76e8 2024-11-14T06:17:08,751 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/c3f9633734724a7a97f03074962f6177 is 50, key is test_row_0/A:col10/1731565027716/Put/seqid=0 2024-11-14T06:17:08,775 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5be089a788a7af479f29a70b78ed76e8#B#compaction#215 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:17:08,775 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/e036f0ebe0ea4ca89ac620ba92234238 is 50, key is test_row_0/B:col10/1731565027556/Put/seqid=0 2024-11-14T06:17:08,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:08,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33182 deadline: 1731565088811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:08,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:08,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:08,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:08,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-14T06:17:08,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:08,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:08,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:08,835 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:08,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:08,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:08,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742084_1260 (size=14341) 2024-11-14T06:17:08,846 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/c3f9633734724a7a97f03074962f6177 2024-11-14T06:17:08,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742083_1259 (size=12207) 2024-11-14T06:17:08,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-14T06:17:08,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742085_1261 (size=12207) 2024-11-14T06:17:08,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/6533f871b537463388e7133a5065cf3d is 50, key is test_row_0/B:col10/1731565027716/Put/seqid=0 2024-11-14T06:17:08,896 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/3bc774442fce4104adfd59e48d4cab74 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/3bc774442fce4104adfd59e48d4cab74 2024-11-14T06:17:08,899 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/e036f0ebe0ea4ca89ac620ba92234238 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/e036f0ebe0ea4ca89ac620ba92234238 2024-11-14T06:17:08,905 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5be089a788a7af479f29a70b78ed76e8/A of 5be089a788a7af479f29a70b78ed76e8 into 3bc774442fce4104adfd59e48d4cab74(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:17:08,905 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5be089a788a7af479f29a70b78ed76e8: 2024-11-14T06:17:08,905 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., storeName=5be089a788a7af479f29a70b78ed76e8/A, priority=13, startTime=1731565028716; duration=0sec 2024-11-14T06:17:08,905 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:17:08,905 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5be089a788a7af479f29a70b78ed76e8:A 2024-11-14T06:17:08,905 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:17:08,907 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:17:08,907 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): 5be089a788a7af479f29a70b78ed76e8/C is initiating minor compaction (all files) 2024-11-14T06:17:08,907 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5be089a788a7af479f29a70b78ed76e8/C in TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:08,908 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/3ed337af46fb4a4db836ed00b65c470d, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/96d322b77c6f4fa9b35173df55d73d58, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/85c1415983534c988b44b310f347c205] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp, totalSize=35.3 K 2024-11-14T06:17:08,908 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ed337af46fb4a4db836ed00b65c470d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731565025631 2024-11-14T06:17:08,909 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96d322b77c6f4fa9b35173df55d73d58, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731565026331 2024-11-14T06:17:08,909 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85c1415983534c988b44b310f347c205, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731565027544 2024-11-14T06:17:08,917 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5be089a788a7af479f29a70b78ed76e8/B of 5be089a788a7af479f29a70b78ed76e8 into e036f0ebe0ea4ca89ac620ba92234238(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:17:08,917 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5be089a788a7af479f29a70b78ed76e8: 2024-11-14T06:17:08,917 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., storeName=5be089a788a7af479f29a70b78ed76e8/B, priority=13, startTime=1731565028717; duration=0sec 2024-11-14T06:17:08,917 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:08,917 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5be089a788a7af479f29a70b78ed76e8:B 2024-11-14T06:17:08,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742086_1262 (size=12001) 2024-11-14T06:17:08,927 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5be089a788a7af479f29a70b78ed76e8#C#compaction#217 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:17:08,927 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/6533f871b537463388e7133a5065cf3d 2024-11-14T06:17:08,928 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/665c3bfd8382464f832c9b0623c270ed is 50, key is test_row_0/C:col10/1731565027556/Put/seqid=0 2024-11-14T06:17:08,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:08,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33182 deadline: 1731565088928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:08,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:08,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:08,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:08,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:08,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33098 deadline: 1731565088937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:08,941 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:08,941 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:08,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:08,941 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:08,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33100 deadline: 1731565088937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:08,942 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:08,942 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:08,942 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:08,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:08,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33126 deadline: 1731565088937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:08,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:08,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:08,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:08,945 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/2b2a7c5a04d74cffa5a8d2cc36374456 is 50, key is test_row_0/C:col10/1731565027716/Put/seqid=0 2024-11-14T06:17:08,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742087_1263 (size=12207) 2024-11-14T06:17:08,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742088_1264 (size=12001) 2024-11-14T06:17:08,976 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/665c3bfd8382464f832c9b0623c270ed as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/665c3bfd8382464f832c9b0623c270ed 2024-11-14T06:17:08,992 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5be089a788a7af479f29a70b78ed76e8/C of 5be089a788a7af479f29a70b78ed76e8 into 665c3bfd8382464f832c9b0623c270ed(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:17:08,992 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5be089a788a7af479f29a70b78ed76e8: 2024-11-14T06:17:08,992 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., storeName=5be089a788a7af479f29a70b78ed76e8/C, priority=13, startTime=1731565028719; duration=0sec 2024-11-14T06:17:08,992 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:08,992 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5be089a788a7af479f29a70b78ed76e8:C 2024-11-14T06:17:08,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-14T06:17:08,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:08,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:08,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:08,999 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:08,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:09,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:09,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:09,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33182 deadline: 1731565089137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:09,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:09,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:09,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:09,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-14T06:17:09,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:09,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:09,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:09,177 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:09,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:09,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:09,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-14T06:17:09,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:09,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:09,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:09,330 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:09,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:09,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:09,384 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/2b2a7c5a04d74cffa5a8d2cc36374456 2024-11-14T06:17:09,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/c3f9633734724a7a97f03074962f6177 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/c3f9633734724a7a97f03074962f6177 2024-11-14T06:17:09,397 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/c3f9633734724a7a97f03074962f6177, entries=200, sequenceid=103, filesize=14.0 K 2024-11-14T06:17:09,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/6533f871b537463388e7133a5065cf3d as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/6533f871b537463388e7133a5065cf3d 2024-11-14T06:17:09,405 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/6533f871b537463388e7133a5065cf3d, entries=150, sequenceid=103, filesize=11.7 K 2024-11-14T06:17:09,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/2b2a7c5a04d74cffa5a8d2cc36374456 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/2b2a7c5a04d74cffa5a8d2cc36374456 2024-11-14T06:17:09,418 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/2b2a7c5a04d74cffa5a8d2cc36374456, entries=150, sequenceid=103, filesize=11.7 K 2024-11-14T06:17:09,420 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 5be089a788a7af479f29a70b78ed76e8 in 683ms, sequenceid=103, compaction requested=false 2024-11-14T06:17:09,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5be089a788a7af479f29a70b78ed76e8: 2024-11-14T06:17:09,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on 5be089a788a7af479f29a70b78ed76e8 2024-11-14T06:17:09,479 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5be089a788a7af479f29a70b78ed76e8 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-14T06:17:09,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=A 2024-11-14T06:17:09,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:09,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=B 2024-11-14T06:17:09,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:09,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=C 2024-11-14T06:17:09,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:09,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/c648b48b8ed049a5a304c1b2dad09e28 is 50, key is test_row_0/A:col10/1731565029477/Put/seqid=0 2024-11-14T06:17:09,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-14T06:17:09,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:09,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:09,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:09,489 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:09,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:09,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:09,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742089_1265 (size=12001) 2024-11-14T06:17:09,495 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/c648b48b8ed049a5a304c1b2dad09e28 2024-11-14T06:17:09,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/7c6ea4143ea8482799dcc85ca510481d is 50, key is test_row_0/B:col10/1731565029477/Put/seqid=0 2024-11-14T06:17:09,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742090_1266 (size=12001) 2024-11-14T06:17:09,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:09,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33182 deadline: 1731565089554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:09,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:09,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:09,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:09,640 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-14T06:17:09,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:09,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:09,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:09,640 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:09,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:09,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:09,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:09,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33182 deadline: 1731565089667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:09,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:09,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:09,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:09,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-14T06:17:09,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:09,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:09,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:09,793 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:09,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:09,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:09,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:09,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33182 deadline: 1731565089877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:09,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:09,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:09,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:09,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-14T06:17:09,908 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/7c6ea4143ea8482799dcc85ca510481d 2024-11-14T06:17:09,917 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/5dee577ec3fc40cd800a7e2b3987c71d is 50, key is test_row_0/C:col10/1731565029477/Put/seqid=0 2024-11-14T06:17:09,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742091_1267 (size=12001) 2024-11-14T06:17:09,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-14T06:17:09,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:09,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:09,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:09,946 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:09,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:09,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:09,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:09,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33098 deadline: 1731565089948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:09,952 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to ef93ef3a83c5,42283,1731564965945 for 'test_row_0' in 5be089a788a7af479f29a70b78ed76e8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T06:17:09,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:09,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:09,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:09,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:09,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33100 deadline: 1731565089958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:09,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:09,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33126 deadline: 1731565089958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:09,960 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to ef93ef3a83c5,42283,1731564965945 for 'test_row_2' in 5be089a788a7af479f29a70b78ed76e8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T06:17:09,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:09,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:09,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:09,961 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to ef93ef3a83c5,42283,1731564965945 for 'test_row_2' in 5be089a788a7af479f29a70b78ed76e8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T06:17:09,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:09,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:09,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:10,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-14T06:17:10,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:10,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:10,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:10,100 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:10,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:10,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:10,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:10,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33182 deadline: 1731565090187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:10,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:10,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:10,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:10,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-14T06:17:10,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:10,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:10,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:10,259 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:10,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:10,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:10,332 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/5dee577ec3fc40cd800a7e2b3987c71d 2024-11-14T06:17:10,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/c648b48b8ed049a5a304c1b2dad09e28 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/c648b48b8ed049a5a304c1b2dad09e28 2024-11-14T06:17:10,353 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/c648b48b8ed049a5a304c1b2dad09e28, entries=150, sequenceid=119, filesize=11.7 K 2024-11-14T06:17:10,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/7c6ea4143ea8482799dcc85ca510481d as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/7c6ea4143ea8482799dcc85ca510481d 2024-11-14T06:17:10,373 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/7c6ea4143ea8482799dcc85ca510481d, entries=150, sequenceid=119, filesize=11.7 K 2024-11-14T06:17:10,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/5dee577ec3fc40cd800a7e2b3987c71d as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/5dee577ec3fc40cd800a7e2b3987c71d 2024-11-14T06:17:10,387 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/5dee577ec3fc40cd800a7e2b3987c71d, entries=150, sequenceid=119, filesize=11.7 K 2024-11-14T06:17:10,390 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 5be089a788a7af479f29a70b78ed76e8 in 911ms, sequenceid=119, compaction requested=true 2024-11-14T06:17:10,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5be089a788a7af479f29a70b78ed76e8: 2024-11-14T06:17:10,390 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:17:10,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5be089a788a7af479f29a70b78ed76e8:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T06:17:10,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:10,391 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:17:10,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5be089a788a7af479f29a70b78ed76e8:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T06:17:10,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:10,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5be089a788a7af479f29a70b78ed76e8:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T06:17:10,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:17:10,398 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38549 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:17:10,398 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): 5be089a788a7af479f29a70b78ed76e8/A is initiating minor compaction (all files) 2024-11-14T06:17:10,399 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:17:10,399 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5be089a788a7af479f29a70b78ed76e8/A in TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:10,399 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): 5be089a788a7af479f29a70b78ed76e8/B is initiating minor compaction (all files) 2024-11-14T06:17:10,399 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5be089a788a7af479f29a70b78ed76e8/B in TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:10,399 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/e036f0ebe0ea4ca89ac620ba92234238, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/6533f871b537463388e7133a5065cf3d, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/7c6ea4143ea8482799dcc85ca510481d] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp, totalSize=35.4 K 2024-11-14T06:17:10,399 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/3bc774442fce4104adfd59e48d4cab74, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/c3f9633734724a7a97f03074962f6177, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/c648b48b8ed049a5a304c1b2dad09e28] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp, totalSize=37.6 K 2024-11-14T06:17:10,400 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting e036f0ebe0ea4ca89ac620ba92234238, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731565027544 2024-11-14T06:17:10,400 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3bc774442fce4104adfd59e48d4cab74, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731565027544 2024-11-14T06:17:10,400 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 6533f871b537463388e7133a5065cf3d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731565027716 2024-11-14T06:17:10,400 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3f9633734724a7a97f03074962f6177, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731565027716 2024-11-14T06:17:10,401 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c6ea4143ea8482799dcc85ca510481d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731565028791 2024-11-14T06:17:10,401 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting c648b48b8ed049a5a304c1b2dad09e28, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731565028791 2024-11-14T06:17:10,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-14T06:17:10,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:10,415 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 5be089a788a7af479f29a70b78ed76e8 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-14T06:17:10,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=A 2024-11-14T06:17:10,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:10,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=B 2024-11-14T06:17:10,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:10,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=C 2024-11-14T06:17:10,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:10,423 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5be089a788a7af479f29a70b78ed76e8#B#compaction#222 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:17:10,423 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/db197aa520d64d10a833d309eee51cf9 is 50, key is test_row_0/B:col10/1731565029477/Put/seqid=0 2024-11-14T06:17:10,425 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5be089a788a7af479f29a70b78ed76e8#A#compaction#223 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:17:10,425 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/4cfe566d261e4eac8ebe79f845120c15 is 50, key is test_row_0/A:col10/1731565029477/Put/seqid=0 2024-11-14T06:17:10,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/1112d835d51549798ed05121f5fa1d8f is 50, key is test_row_0/A:col10/1731565029526/Put/seqid=0 2024-11-14T06:17:10,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742093_1269 (size=12309) 2024-11-14T06:17:10,508 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/4cfe566d261e4eac8ebe79f845120c15 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/4cfe566d261e4eac8ebe79f845120c15 2024-11-14T06:17:10,519 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5be089a788a7af479f29a70b78ed76e8/A of 5be089a788a7af479f29a70b78ed76e8 into 4cfe566d261e4eac8ebe79f845120c15(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:17:10,519 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5be089a788a7af479f29a70b78ed76e8: 2024-11-14T06:17:10,519 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., storeName=5be089a788a7af479f29a70b78ed76e8/A, priority=13, startTime=1731565030390; duration=0sec 2024-11-14T06:17:10,519 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:17:10,519 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5be089a788a7af479f29a70b78ed76e8:A 2024-11-14T06:17:10,519 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:17:10,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742094_1270 (size=12151) 2024-11-14T06:17:10,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742092_1268 (size=12309) 2024-11-14T06:17:10,527 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/1112d835d51549798ed05121f5fa1d8f 2024-11-14T06:17:10,529 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:17:10,529 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): 5be089a788a7af479f29a70b78ed76e8/C is initiating minor compaction (all files) 2024-11-14T06:17:10,529 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5be089a788a7af479f29a70b78ed76e8/C in TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:10,529 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/665c3bfd8382464f832c9b0623c270ed, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/2b2a7c5a04d74cffa5a8d2cc36374456, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/5dee577ec3fc40cd800a7e2b3987c71d] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp, totalSize=35.4 K 2024-11-14T06:17:10,531 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 665c3bfd8382464f832c9b0623c270ed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731565027544 2024-11-14T06:17:10,532 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b2a7c5a04d74cffa5a8d2cc36374456, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731565027716 2024-11-14T06:17:10,533 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5dee577ec3fc40cd800a7e2b3987c71d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731565028791 2024-11-14T06:17:10,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/4916ba8c4d7b4f8599da02e34c241847 is 50, key is test_row_0/B:col10/1731565029526/Put/seqid=0 2024-11-14T06:17:10,592 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5be089a788a7af479f29a70b78ed76e8#C#compaction#226 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:17:10,593 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/78a988e0a4e74766a1dc6bebfa904c48 is 50, key is test_row_0/C:col10/1731565029477/Put/seqid=0 2024-11-14T06:17:10,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742095_1271 (size=12151) 2024-11-14T06:17:10,634 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/4916ba8c4d7b4f8599da02e34c241847 2024-11-14T06:17:10,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742096_1272 (size=12309) 2024-11-14T06:17:10,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/19602ce83a5b4b6abfbccfee4eaa41f6 is 50, key is test_row_0/C:col10/1731565029526/Put/seqid=0 2024-11-14T06:17:10,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742097_1273 (size=12151) 2024-11-14T06:17:10,666 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/19602ce83a5b4b6abfbccfee4eaa41f6 2024-11-14T06:17:10,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/1112d835d51549798ed05121f5fa1d8f as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/1112d835d51549798ed05121f5fa1d8f 2024-11-14T06:17:10,678 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/1112d835d51549798ed05121f5fa1d8f, entries=150, sequenceid=138, filesize=11.9 K 2024-11-14T06:17:10,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/4916ba8c4d7b4f8599da02e34c241847 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/4916ba8c4d7b4f8599da02e34c241847 2024-11-14T06:17:10,685 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/4916ba8c4d7b4f8599da02e34c241847, entries=150, sequenceid=138, filesize=11.9 K 2024-11-14T06:17:10,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/19602ce83a5b4b6abfbccfee4eaa41f6 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/19602ce83a5b4b6abfbccfee4eaa41f6 2024-11-14T06:17:10,692 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/19602ce83a5b4b6abfbccfee4eaa41f6, entries=150, sequenceid=138, filesize=11.9 K 2024-11-14T06:17:10,693 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=0 B/0 for 5be089a788a7af479f29a70b78ed76e8 in 278ms, sequenceid=138, compaction requested=false 2024-11-14T06:17:10,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 5be089a788a7af479f29a70b78ed76e8: 2024-11-14T06:17:10,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:10,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-14T06:17:10,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-14T06:17:10,697 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-14T06:17:10,697 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9530 sec 2024-11-14T06:17:10,702 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 2.9600 sec 2024-11-14T06:17:10,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on 5be089a788a7af479f29a70b78ed76e8 2024-11-14T06:17:10,738 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5be089a788a7af479f29a70b78ed76e8 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T06:17:10,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=A 2024-11-14T06:17:10,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:10,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=B 2024-11-14T06:17:10,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:10,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=C 2024-11-14T06:17:10,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:10,745 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/1cf990348de44708b684ec68c81fd52c is 50, key is test_row_0/A:col10/1731565030721/Put/seqid=0 2024-11-14T06:17:10,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742098_1274 (size=14541) 2024-11-14T06:17:10,788 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/1cf990348de44708b684ec68c81fd52c 2024-11-14T06:17:10,795 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/aa13b41b14224992bf0cc31f447f4295 is 50, key is test_row_0/B:col10/1731565030721/Put/seqid=0 2024-11-14T06:17:10,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742099_1275 (size=12151) 2024-11-14T06:17:10,815 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/aa13b41b14224992bf0cc31f447f4295 2024-11-14T06:17:10,829 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/8ed7c76aeffd47eda3e5a23a56db55fa is 50, key is test_row_0/C:col10/1731565030721/Put/seqid=0 2024-11-14T06:17:10,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742100_1276 (size=12151) 2024-11-14T06:17:10,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731565030821, max timestamp=1731565030853Num uniques 859; 2024-11-14T06:17:10,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731565030821, max timestamp=1731565030853Num uniques 859; 2024-11-14T06:17:10,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731565030821, max timestamp=1731565030853Num uniques 859; 2024-11-14T06:17:10,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T06:17:10,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T06:17:10,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T06:17:10,931 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/db197aa520d64d10a833d309eee51cf9 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/db197aa520d64d10a833d309eee51cf9 2024-11-14T06:17:10,936 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5be089a788a7af479f29a70b78ed76e8/B of 5be089a788a7af479f29a70b78ed76e8 into db197aa520d64d10a833d309eee51cf9(size=12.0 K), total size for store is 23.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:17:10,936 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5be089a788a7af479f29a70b78ed76e8: 2024-11-14T06:17:10,936 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., storeName=5be089a788a7af479f29a70b78ed76e8/B, priority=13, startTime=1731565030391; duration=0sec 2024-11-14T06:17:10,936 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:10,936 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5be089a788a7af479f29a70b78ed76e8:B 2024-11-14T06:17:10,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:10,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33182 deadline: 1731565090948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:10,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:10,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:10,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:11,050 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/78a988e0a4e74766a1dc6bebfa904c48 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/78a988e0a4e74766a1dc6bebfa904c48 2024-11-14T06:17:11,054 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5be089a788a7af479f29a70b78ed76e8/C of 5be089a788a7af479f29a70b78ed76e8 into 78a988e0a4e74766a1dc6bebfa904c48(size=12.0 K), total size for store is 23.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:17:11,055 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5be089a788a7af479f29a70b78ed76e8: 2024-11-14T06:17:11,055 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., storeName=5be089a788a7af479f29a70b78ed76e8/C, priority=13, startTime=1731565030391; duration=0sec 2024-11-14T06:17:11,055 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:11,055 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5be089a788a7af479f29a70b78ed76e8:C 2024-11-14T06:17:11,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:11,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33182 deadline: 1731565091058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:11,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:11,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:11,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:11,236 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/8ed7c76aeffd47eda3e5a23a56db55fa 2024-11-14T06:17:11,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/1cf990348de44708b684ec68c81fd52c as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/1cf990348de44708b684ec68c81fd52c 2024-11-14T06:17:11,253 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/1cf990348de44708b684ec68c81fd52c, entries=200, sequenceid=149, filesize=14.2 K 2024-11-14T06:17:11,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/aa13b41b14224992bf0cc31f447f4295 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/aa13b41b14224992bf0cc31f447f4295 2024-11-14T06:17:11,259 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/aa13b41b14224992bf0cc31f447f4295, entries=150, sequenceid=149, filesize=11.9 K 2024-11-14T06:17:11,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/8ed7c76aeffd47eda3e5a23a56db55fa as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/8ed7c76aeffd47eda3e5a23a56db55fa 2024-11-14T06:17:11,265 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/8ed7c76aeffd47eda3e5a23a56db55fa, entries=150, sequenceid=149, filesize=11.9 K 2024-11-14T06:17:11,266 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 5be089a788a7af479f29a70b78ed76e8 in 527ms, sequenceid=149, compaction requested=true 2024-11-14T06:17:11,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5be089a788a7af479f29a70b78ed76e8: 2024-11-14T06:17:11,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5be089a788a7af479f29a70b78ed76e8:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T06:17:11,266 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:17:11,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:11,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5be089a788a7af479f29a70b78ed76e8:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T06:17:11,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:11,266 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:17:11,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5be089a788a7af479f29a70b78ed76e8:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T06:17:11,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:17:11,267 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:17:11,267 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): 5be089a788a7af479f29a70b78ed76e8/B is initiating minor compaction (all files) 2024-11-14T06:17:11,267 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5be089a788a7af479f29a70b78ed76e8/B in TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:11,267 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/db197aa520d64d10a833d309eee51cf9, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/4916ba8c4d7b4f8599da02e34c241847, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/aa13b41b14224992bf0cc31f447f4295] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp, totalSize=35.8 K 2024-11-14T06:17:11,267 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39001 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:17:11,267 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): 5be089a788a7af479f29a70b78ed76e8/A is initiating minor compaction (all files) 2024-11-14T06:17:11,267 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5be089a788a7af479f29a70b78ed76e8/A in TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:11,267 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/4cfe566d261e4eac8ebe79f845120c15, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/1112d835d51549798ed05121f5fa1d8f, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/1cf990348de44708b684ec68c81fd52c] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp, totalSize=38.1 K 2024-11-14T06:17:11,268 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting db197aa520d64d10a833d309eee51cf9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731565028791 2024-11-14T06:17:11,268 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4cfe566d261e4eac8ebe79f845120c15, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731565028791 2024-11-14T06:17:11,268 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 4916ba8c4d7b4f8599da02e34c241847, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731565029526 2024-11-14T06:17:11,268 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1112d835d51549798ed05121f5fa1d8f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731565029526 2024-11-14T06:17:11,269 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting aa13b41b14224992bf0cc31f447f4295, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1731565030701 2024-11-14T06:17:11,269 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1cf990348de44708b684ec68c81fd52c, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1731565030701 2024-11-14T06:17:11,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on 5be089a788a7af479f29a70b78ed76e8 2024-11-14T06:17:11,270 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5be089a788a7af479f29a70b78ed76e8 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-14T06:17:11,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=A 2024-11-14T06:17:11,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-14T06:17:11,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=B 2024-11-14T06:17:11,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-14T06:17:11,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=C 2024-11-14T06:17:11,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-14T06:17:11,281 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5be089a788a7af479f29a70b78ed76e8#A#compaction#231 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:17:11,281 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/7eba4ce2914b45549c5ec0a43f705cff is 50, key is test_row_0/A:col10/1731565030721/Put/seqid=0 2024-11-14T06:17:11,283 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5be089a788a7af479f29a70b78ed76e8#B#compaction#232 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:17:11,284 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/256b8548ce5843af803b74ae0b39bc9c is 50, key is test_row_0/B:col10/1731565030721/Put/seqid=0 2024-11-14T06:17:11,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/89fb7ac2abe54dccaf563a00eb1cdef7 is 50, key is test_row_0/A:col10/1731565031268/Put/seqid=0 2024-11-14T06:17:11,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742102_1278 (size=12561) 2024-11-14T06:17:11,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742101_1277 (size=12561) 2024-11-14T06:17:11,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742103_1279 (size=12151) 2024-11-14T06:17:11,319 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/89fb7ac2abe54dccaf563a00eb1cdef7 2024-11-14T06:17:11,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:11,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33182 deadline: 1731565091314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:11,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:11,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:11,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:11,328 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/1d4feadb9469403d8a6134285273be96 is 50, key is test_row_0/B:col10/1731565031268/Put/seqid=0 2024-11-14T06:17:11,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742104_1280 (size=12151) 2024-11-14T06:17:11,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:11,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33182 deadline: 1731565091427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:11,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:11,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:11,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:11,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:11,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33182 deadline: 1731565091637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:11,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:11,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:11,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:11,716 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/256b8548ce5843af803b74ae0b39bc9c as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/256b8548ce5843af803b74ae0b39bc9c 2024-11-14T06:17:11,722 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/7eba4ce2914b45549c5ec0a43f705cff as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/7eba4ce2914b45549c5ec0a43f705cff 2024-11-14T06:17:11,725 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5be089a788a7af479f29a70b78ed76e8/B of 5be089a788a7af479f29a70b78ed76e8 into 256b8548ce5843af803b74ae0b39bc9c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:17:11,725 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5be089a788a7af479f29a70b78ed76e8: 2024-11-14T06:17:11,726 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., storeName=5be089a788a7af479f29a70b78ed76e8/B, priority=13, startTime=1731565031266; duration=0sec 2024-11-14T06:17:11,726 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:17:11,726 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5be089a788a7af479f29a70b78ed76e8:B 2024-11-14T06:17:11,726 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:17:11,727 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:17:11,727 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): 5be089a788a7af479f29a70b78ed76e8/C is initiating minor compaction (all files) 2024-11-14T06:17:11,727 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5be089a788a7af479f29a70b78ed76e8/C in TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:11,728 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/78a988e0a4e74766a1dc6bebfa904c48, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/19602ce83a5b4b6abfbccfee4eaa41f6, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/8ed7c76aeffd47eda3e5a23a56db55fa] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp, totalSize=35.8 K 2024-11-14T06:17:11,729 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 78a988e0a4e74766a1dc6bebfa904c48, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731565028791 2024-11-14T06:17:11,730 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 19602ce83a5b4b6abfbccfee4eaa41f6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731565029526 2024-11-14T06:17:11,731 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ed7c76aeffd47eda3e5a23a56db55fa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1731565030701 2024-11-14T06:17:11,732 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5be089a788a7af479f29a70b78ed76e8/A of 5be089a788a7af479f29a70b78ed76e8 into 7eba4ce2914b45549c5ec0a43f705cff(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:17:11,732 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5be089a788a7af479f29a70b78ed76e8: 2024-11-14T06:17:11,732 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., storeName=5be089a788a7af479f29a70b78ed76e8/A, priority=13, startTime=1731565031266; duration=0sec 2024-11-14T06:17:11,732 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:11,732 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5be089a788a7af479f29a70b78ed76e8:A 2024-11-14T06:17:11,749 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/1d4feadb9469403d8a6134285273be96 2024-11-14T06:17:11,766 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5be089a788a7af479f29a70b78ed76e8#C#compaction#235 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:17:11,766 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/9032b2a6acd14efea98a0f07c01d566b is 50, key is test_row_0/C:col10/1731565030721/Put/seqid=0 2024-11-14T06:17:11,781 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/31cbf3aee4b04d78960b24266b6e31f5 is 50, key is test_row_0/C:col10/1731565031268/Put/seqid=0 2024-11-14T06:17:11,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742105_1281 (size=12561) 2024-11-14T06:17:11,849 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/9032b2a6acd14efea98a0f07c01d566b as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/9032b2a6acd14efea98a0f07c01d566b 2024-11-14T06:17:11,855 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5be089a788a7af479f29a70b78ed76e8/C of 5be089a788a7af479f29a70b78ed76e8 into 9032b2a6acd14efea98a0f07c01d566b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:17:11,855 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5be089a788a7af479f29a70b78ed76e8: 2024-11-14T06:17:11,856 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., storeName=5be089a788a7af479f29a70b78ed76e8/C, priority=13, startTime=1731565031266; duration=0sec 2024-11-14T06:17:11,856 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:11,856 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5be089a788a7af479f29a70b78ed76e8:C 2024-11-14T06:17:11,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742106_1282 (size=12151) 2024-11-14T06:17:11,860 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/31cbf3aee4b04d78960b24266b6e31f5 2024-11-14T06:17:11,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/89fb7ac2abe54dccaf563a00eb1cdef7 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/89fb7ac2abe54dccaf563a00eb1cdef7 2024-11-14T06:17:11,873 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/89fb7ac2abe54dccaf563a00eb1cdef7, entries=150, sequenceid=189, filesize=11.9 K 2024-11-14T06:17:11,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/1d4feadb9469403d8a6134285273be96 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/1d4feadb9469403d8a6134285273be96 2024-11-14T06:17:11,888 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/1d4feadb9469403d8a6134285273be96, entries=150, sequenceid=189, filesize=11.9 K 2024-11-14T06:17:11,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/31cbf3aee4b04d78960b24266b6e31f5 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/31cbf3aee4b04d78960b24266b6e31f5 2024-11-14T06:17:11,897 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/31cbf3aee4b04d78960b24266b6e31f5, entries=150, sequenceid=189, filesize=11.9 K 2024-11-14T06:17:11,899 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 5be089a788a7af479f29a70b78ed76e8 in 629ms, sequenceid=189, compaction requested=false 2024-11-14T06:17:11,899 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5be089a788a7af479f29a70b78ed76e8: 2024-11-14T06:17:11,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-14T06:17:11,918 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:17:11,919 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T06:17:11,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-14T06:17:11,922 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T06:17:11,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-14T06:17:11,924 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T06:17:11,924 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T06:17:11,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on 5be089a788a7af479f29a70b78ed76e8 2024-11-14T06:17:11,951 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5be089a788a7af479f29a70b78ed76e8 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T06:17:11,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=A 2024-11-14T06:17:11,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:11,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=B 2024-11-14T06:17:11,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:11,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=C 2024-11-14T06:17:11,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:11,961 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/88733b189d104e188694c396dc2f5c29 is 50, key is test_row_0/A:col10/1731565031312/Put/seqid=0 2024-11-14T06:17:11,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742107_1283 (size=9757) 2024-11-14T06:17:12,002 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731565031971, max timestamp=1731565032000Num uniques 859; 2024-11-14T06:17:12,002 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T06:17:12,003 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731565031971, max timestamp=1731565032000Num uniques 859; 2024-11-14T06:17:12,003 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T06:17:12,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:12,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33126 deadline: 1731565092018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:12,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:12,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:12,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:12,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-14T06:17:12,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:12,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33182 deadline: 1731565092020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:12,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:12,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:12,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:12,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33098 deadline: 1731565092021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:12,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:12,028 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:12,028 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:12,028 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:12,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:12,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33100 deadline: 1731565092024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:12,030 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:12,030 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:12,030 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:12,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-14T06:17:12,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:12,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:12,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:12,080 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:12,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:12,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:12,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:12,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33126 deadline: 1731565092127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:12,131 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:12,131 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:12,131 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:12,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:12,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33182 deadline: 1731565092138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:12,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:12,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:12,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:12,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33100 deadline: 1731565092138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:12,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:12,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:12,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:12,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:12,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:12,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33098 deadline: 1731565092138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:12,142 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:12,142 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:12,142 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:12,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-14T06:17:12,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:12,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:12,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:12,235 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:12,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:12,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:12,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-14T06:17:12,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:12,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33126 deadline: 1731565092337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:12,342 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:12,342 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:12,342 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:12,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:12,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33182 deadline: 1731565092347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:12,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:12,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:12,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:12,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:12,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33098 deadline: 1731565092347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:12,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:12,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:12,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:12,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:12,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33100 deadline: 1731565092347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:12,348 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:12,348 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:12,348 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:12,370 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/88733b189d104e188694c396dc2f5c29 2024-11-14T06:17:12,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/dc96c4c8eefb4e01ba44960721538c59 is 50, key is test_row_0/B:col10/1731565031312/Put/seqid=0 2024-11-14T06:17:12,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742108_1284 (size=9757) 2024-11-14T06:17:12,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-14T06:17:12,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:12,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:12,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:12,392 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:12,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:12,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:12,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-14T06:17:12,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:12,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:12,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:12,546 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:12,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:12,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:12,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-14T06:17:12,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:12,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33126 deadline: 1731565092647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:12,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:12,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:12,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:12,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:12,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33098 deadline: 1731565092658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:12,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:12,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:12,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:12,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33182 deadline: 1731565092658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:12,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:12,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:12,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:12,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:12,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:12,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33100 deadline: 1731565092662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:12,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:12,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:12,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:12,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-14T06:17:12,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:12,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:12,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:12,702 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:12,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:12,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:12,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:12,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33168 deadline: 1731565092747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:12,751 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to ef93ef3a83c5,42283,1731564965945 for 'test_row_1' in 5be089a788a7af479f29a70b78ed76e8 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8276 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T06:17:12,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:12,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:12,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:12,783 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/dc96c4c8eefb4e01ba44960721538c59 2024-11-14T06:17:12,792 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/6f40653eb7284562bd5bd28a3dab7a6b is 50, key is test_row_0/C:col10/1731565031312/Put/seqid=0 2024-11-14T06:17:12,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742109_1285 (size=9757) 2024-11-14T06:17:12,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-14T06:17:12,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:12,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:12,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:12,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:12,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:12,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:13,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-14T06:17:13,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:13,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:13,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:13,008 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:13,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:13,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:13,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-14T06:17:13,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:13,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33126 deadline: 1731565093157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:13,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:13,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:13,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:13,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-14T06:17:13,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:13,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:13,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:13,162 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:13,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:13,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:13,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:13,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33098 deadline: 1731565093167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:13,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:13,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:13,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33182 deadline: 1731565093167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:13,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:13,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:13,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:13,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:13,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:13,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:13,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33100 deadline: 1731565093177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:13,179 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:13,179 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:13,179 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:13,198 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/6f40653eb7284562bd5bd28a3dab7a6b 2024-11-14T06:17:13,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/88733b189d104e188694c396dc2f5c29 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/88733b189d104e188694c396dc2f5c29 2024-11-14T06:17:13,208 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/88733b189d104e188694c396dc2f5c29, entries=100, sequenceid=202, filesize=9.5 K 2024-11-14T06:17:13,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/dc96c4c8eefb4e01ba44960721538c59 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/dc96c4c8eefb4e01ba44960721538c59 2024-11-14T06:17:13,213 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/dc96c4c8eefb4e01ba44960721538c59, entries=100, sequenceid=202, filesize=9.5 K 2024-11-14T06:17:13,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/6f40653eb7284562bd5bd28a3dab7a6b as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/6f40653eb7284562bd5bd28a3dab7a6b 2024-11-14T06:17:13,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/6f40653eb7284562bd5bd28a3dab7a6b, entries=100, sequenceid=202, filesize=9.5 K 2024-11-14T06:17:13,218 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=131.94 KB/135110 for 5be089a788a7af479f29a70b78ed76e8 in 1268ms, sequenceid=202, compaction requested=true 2024-11-14T06:17:13,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5be089a788a7af479f29a70b78ed76e8: 2024-11-14T06:17:13,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5be089a788a7af479f29a70b78ed76e8:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T06:17:13,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:13,218 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:17:13,218 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:17:13,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5be089a788a7af479f29a70b78ed76e8:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T06:17:13,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:13,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5be089a788a7af479f29a70b78ed76e8:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T06:17:13,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:17:13,219 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34469 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:17:13,219 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34469 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:17:13,219 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): 5be089a788a7af479f29a70b78ed76e8/B is initiating minor compaction (all files) 2024-11-14T06:17:13,219 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): 5be089a788a7af479f29a70b78ed76e8/A is initiating minor compaction (all files) 2024-11-14T06:17:13,219 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5be089a788a7af479f29a70b78ed76e8/A in TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:13,219 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5be089a788a7af479f29a70b78ed76e8/B in TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:13,220 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/7eba4ce2914b45549c5ec0a43f705cff, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/89fb7ac2abe54dccaf563a00eb1cdef7, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/88733b189d104e188694c396dc2f5c29] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp, totalSize=33.7 K 2024-11-14T06:17:13,220 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/256b8548ce5843af803b74ae0b39bc9c, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/1d4feadb9469403d8a6134285273be96, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/dc96c4c8eefb4e01ba44960721538c59] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp, totalSize=33.7 K 2024-11-14T06:17:13,220 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 256b8548ce5843af803b74ae0b39bc9c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1731565030701 2024-11-14T06:17:13,220 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7eba4ce2914b45549c5ec0a43f705cff, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1731565030701 2024-11-14T06:17:13,220 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89fb7ac2abe54dccaf563a00eb1cdef7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731565030931 2024-11-14T06:17:13,221 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d4feadb9469403d8a6134285273be96, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731565030931 2024-11-14T06:17:13,224 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88733b189d104e188694c396dc2f5c29, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731565031312 2024-11-14T06:17:13,224 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting dc96c4c8eefb4e01ba44960721538c59, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731565031312 2024-11-14T06:17:13,238 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5be089a788a7af479f29a70b78ed76e8#B#compaction#240 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:17:13,239 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/3e7ec646d6554540b5165a57e008e3a0 is 50, key is test_row_0/B:col10/1731565031312/Put/seqid=0 2024-11-14T06:17:13,242 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5be089a788a7af479f29a70b78ed76e8#A#compaction#241 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:17:13,243 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/dd0690d3e5df4df6b54a3d2786f31ec0 is 50, key is test_row_0/A:col10/1731565031312/Put/seqid=0 2024-11-14T06:17:13,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742110_1286 (size=12663) 2024-11-14T06:17:13,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742111_1287 (size=12663) 2024-11-14T06:17:13,314 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-14T06:17:13,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:13,314 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 5be089a788a7af479f29a70b78ed76e8 3/3 column families, dataSize=131.94 KB heapSize=383.31 KB 2024-11-14T06:17:13,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=A 2024-11-14T06:17:13,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:17:13,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=B 2024-11-14T06:17:13,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:17:13,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=C 2024-11-14T06:17:13,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:17:13,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/3d56ddde44f64fbb9180ed136cf90a88 is 50, key is test_row_0/A:col10/1731565032021/Put/seqid=0 2024-11-14T06:17:13,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742112_1288 (size=12151) 2024-11-14T06:17:13,673 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/3e7ec646d6554540b5165a57e008e3a0 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/3e7ec646d6554540b5165a57e008e3a0 2024-11-14T06:17:13,679 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5be089a788a7af479f29a70b78ed76e8/B of 5be089a788a7af479f29a70b78ed76e8 into 3e7ec646d6554540b5165a57e008e3a0(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:17:13,679 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5be089a788a7af479f29a70b78ed76e8: 2024-11-14T06:17:13,679 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., storeName=5be089a788a7af479f29a70b78ed76e8/B, priority=13, startTime=1731565033218; duration=0sec 2024-11-14T06:17:13,679 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:17:13,679 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5be089a788a7af479f29a70b78ed76e8:B 2024-11-14T06:17:13,679 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:17:13,681 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34469 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:17:13,681 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): 5be089a788a7af479f29a70b78ed76e8/C is initiating minor compaction (all files) 2024-11-14T06:17:13,681 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5be089a788a7af479f29a70b78ed76e8/C in TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:13,681 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/9032b2a6acd14efea98a0f07c01d566b, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/31cbf3aee4b04d78960b24266b6e31f5, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/6f40653eb7284562bd5bd28a3dab7a6b] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp, totalSize=33.7 K 2024-11-14T06:17:13,682 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 9032b2a6acd14efea98a0f07c01d566b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1731565030701 2024-11-14T06:17:13,682 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 31cbf3aee4b04d78960b24266b6e31f5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731565030931 2024-11-14T06:17:13,682 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f40653eb7284562bd5bd28a3dab7a6b, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731565031312 2024-11-14T06:17:13,693 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/dd0690d3e5df4df6b54a3d2786f31ec0 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/dd0690d3e5df4df6b54a3d2786f31ec0 2024-11-14T06:17:13,702 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5be089a788a7af479f29a70b78ed76e8/A of 5be089a788a7af479f29a70b78ed76e8 into dd0690d3e5df4df6b54a3d2786f31ec0(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:17:13,702 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5be089a788a7af479f29a70b78ed76e8: 2024-11-14T06:17:13,702 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., storeName=5be089a788a7af479f29a70b78ed76e8/A, priority=13, startTime=1731565033218; duration=0sec 2024-11-14T06:17:13,702 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:13,702 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5be089a788a7af479f29a70b78ed76e8:A 2024-11-14T06:17:13,706 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5be089a788a7af479f29a70b78ed76e8#C#compaction#243 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:17:13,706 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/b39c10af3b7b4786a72c00ce337c31ec is 50, key is test_row_0/C:col10/1731565031312/Put/seqid=0 2024-11-14T06:17:13,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742113_1289 (size=12663) 2024-11-14T06:17:13,736 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/3d56ddde44f64fbb9180ed136cf90a88 2024-11-14T06:17:13,746 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/b39c10af3b7b4786a72c00ce337c31ec as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/b39c10af3b7b4786a72c00ce337c31ec 2024-11-14T06:17:13,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/adbe6f8c3d69443798f3707475f993e3 is 50, key is test_row_0/B:col10/1731565032021/Put/seqid=0 2024-11-14T06:17:13,751 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5be089a788a7af479f29a70b78ed76e8/C of 5be089a788a7af479f29a70b78ed76e8 into b39c10af3b7b4786a72c00ce337c31ec(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:17:13,751 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5be089a788a7af479f29a70b78ed76e8: 2024-11-14T06:17:13,751 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., storeName=5be089a788a7af479f29a70b78ed76e8/C, priority=13, startTime=1731565033218; duration=0sec 2024-11-14T06:17:13,751 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:13,751 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5be089a788a7af479f29a70b78ed76e8:C 2024-11-14T06:17:13,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742114_1290 (size=12151) 2024-11-14T06:17:13,765 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/adbe6f8c3d69443798f3707475f993e3 2024-11-14T06:17:13,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/6c0388b8085c45d48008b46d44d9c432 is 50, key is test_row_0/C:col10/1731565032021/Put/seqid=0 2024-11-14T06:17:13,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742115_1291 (size=12151) 2024-11-14T06:17:14,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-14T06:17:14,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on 5be089a788a7af479f29a70b78ed76e8 2024-11-14T06:17:14,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:14,190 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/6c0388b8085c45d48008b46d44d9c432 2024-11-14T06:17:14,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:14,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33182 deadline: 1731565094186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:14,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:14,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33098 deadline: 1731565094187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:14,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:14,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:14,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:14,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:14,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33100 deadline: 1731565094187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:14,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:14,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33126 deadline: 1731565094189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:14,192 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to ef93ef3a83c5,42283,1731564965945 for 'test_row_2' in 5be089a788a7af479f29a70b78ed76e8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T06:17:14,192 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to ef93ef3a83c5,42283,1731564965945 for 'test_row_2' in 5be089a788a7af479f29a70b78ed76e8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T06:17:14,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:14,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:14,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:14,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:14,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:14,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:14,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:14,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:14,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:14,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/3d56ddde44f64fbb9180ed136cf90a88 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/3d56ddde44f64fbb9180ed136cf90a88 2024-11-14T06:17:14,203 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/3d56ddde44f64fbb9180ed136cf90a88, entries=150, sequenceid=234, filesize=11.9 K 2024-11-14T06:17:14,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/adbe6f8c3d69443798f3707475f993e3 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/adbe6f8c3d69443798f3707475f993e3 2024-11-14T06:17:14,213 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/adbe6f8c3d69443798f3707475f993e3, entries=150, sequenceid=234, filesize=11.9 K 2024-11-14T06:17:14,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/6c0388b8085c45d48008b46d44d9c432 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/6c0388b8085c45d48008b46d44d9c432 2024-11-14T06:17:14,219 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/6c0388b8085c45d48008b46d44d9c432, entries=150, sequenceid=234, filesize=11.9 K 2024-11-14T06:17:14,222 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~131.94 KB/135110, heapSize ~383.27 KB/392464, currentSize=46.96 KB/48090 for 5be089a788a7af479f29a70b78ed76e8 in 908ms, sequenceid=234, compaction requested=false 2024-11-14T06:17:14,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 5be089a788a7af479f29a70b78ed76e8: 2024-11-14T06:17:14,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:14,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-14T06:17:14,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-14T06:17:14,225 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-14T06:17:14,225 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2990 sec 2024-11-14T06:17:14,226 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 2.3060 sec 2024-11-14T06:17:14,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on 5be089a788a7af479f29a70b78ed76e8 2024-11-14T06:17:14,300 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5be089a788a7af479f29a70b78ed76e8 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-14T06:17:14,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=A 2024-11-14T06:17:14,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:14,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=B 2024-11-14T06:17:14,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:14,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=C 2024-11-14T06:17:14,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:14,310 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/d357c45b80ad409d9bc1a94b03a2e775 is 50, key is test_row_0/A:col10/1731565034299/Put/seqid=0 2024-11-14T06:17:14,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742116_1292 (size=12151) 2024-11-14T06:17:14,331 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/d357c45b80ad409d9bc1a94b03a2e775 2024-11-14T06:17:14,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/0b7ab2edf33945bea542a47cd08a1c57 is 50, key is test_row_0/B:col10/1731565034299/Put/seqid=0 2024-11-14T06:17:14,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742117_1293 (size=12151) 2024-11-14T06:17:14,343 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/0b7ab2edf33945bea542a47cd08a1c57 2024-11-14T06:17:14,351 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731565034327, max timestamp=1731565034348Num uniques 859; 2024-11-14T06:17:14,351 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T06:17:14,353 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/9e492613c9a14aa6ae215a1461804e34 is 50, key is test_row_0/C:col10/1731565034299/Put/seqid=0 2024-11-14T06:17:14,353 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731565034327, max timestamp=1731565034348Num uniques 859; 2024-11-14T06:17:14,353 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-14T06:17:14,356 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731565034327, max timestamp=1731565034348Num uniques 859; 2024-11-14T06:17:14,356 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-14T06:17:14,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742118_1294 (size=12151) 2024-11-14T06:17:14,364 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/9e492613c9a14aa6ae215a1461804e34 2024-11-14T06:17:14,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/d357c45b80ad409d9bc1a94b03a2e775 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/d357c45b80ad409d9bc1a94b03a2e775 2024-11-14T06:17:14,373 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/d357c45b80ad409d9bc1a94b03a2e775, entries=150, sequenceid=249, filesize=11.9 K 2024-11-14T06:17:14,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/0b7ab2edf33945bea542a47cd08a1c57 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/0b7ab2edf33945bea542a47cd08a1c57 2024-11-14T06:17:14,379 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/0b7ab2edf33945bea542a47cd08a1c57, entries=150, sequenceid=249, filesize=11.9 K 2024-11-14T06:17:14,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/9e492613c9a14aa6ae215a1461804e34 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/9e492613c9a14aa6ae215a1461804e34 2024-11-14T06:17:14,384 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/9e492613c9a14aa6ae215a1461804e34, entries=150, sequenceid=249, filesize=11.9 K 2024-11-14T06:17:14,403 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=75.25 KB/77052 for 5be089a788a7af479f29a70b78ed76e8 in 103ms, sequenceid=249, compaction requested=true 2024-11-14T06:17:14,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5be089a788a7af479f29a70b78ed76e8: 2024-11-14T06:17:14,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5be089a788a7af479f29a70b78ed76e8:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T06:17:14,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:14,403 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:17:14,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5be089a788a7af479f29a70b78ed76e8:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T06:17:14,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:14,403 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:17:14,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5be089a788a7af479f29a70b78ed76e8:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T06:17:14,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on 5be089a788a7af479f29a70b78ed76e8 2024-11-14T06:17:14,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:17:14,404 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5be089a788a7af479f29a70b78ed76e8 3/3 column families, dataSize=88.66 KB heapSize=248.78 KB 2024-11-14T06:17:14,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=A 2024-11-14T06:17:14,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:17:14,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=B 2024-11-14T06:17:14,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:17:14,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=C 2024-11-14T06:17:14,404 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:17:14,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:17:14,405 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): 5be089a788a7af479f29a70b78ed76e8/B is initiating minor compaction (all files) 2024-11-14T06:17:14,405 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5be089a788a7af479f29a70b78ed76e8/B in TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:14,405 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/3e7ec646d6554540b5165a57e008e3a0, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/adbe6f8c3d69443798f3707475f993e3, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/0b7ab2edf33945bea542a47cd08a1c57] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp, totalSize=36.1 K 2024-11-14T06:17:14,405 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:17:14,405 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e7ec646d6554540b5165a57e008e3a0, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731565030931 2024-11-14T06:17:14,406 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): 5be089a788a7af479f29a70b78ed76e8/A is initiating minor compaction (all files) 2024-11-14T06:17:14,406 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5be089a788a7af479f29a70b78ed76e8/A in TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:14,406 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/dd0690d3e5df4df6b54a3d2786f31ec0, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/3d56ddde44f64fbb9180ed136cf90a88, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/d357c45b80ad409d9bc1a94b03a2e775] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp, totalSize=36.1 K 2024-11-14T06:17:14,406 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting adbe6f8c3d69443798f3707475f993e3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731565032016 2024-11-14T06:17:14,406 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd0690d3e5df4df6b54a3d2786f31ec0, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731565030931 2024-11-14T06:17:14,406 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b7ab2edf33945bea542a47cd08a1c57, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731565034187 2024-11-14T06:17:14,407 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d56ddde44f64fbb9180ed136cf90a88, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731565032016 2024-11-14T06:17:14,407 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting d357c45b80ad409d9bc1a94b03a2e775, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731565034187 2024-11-14T06:17:14,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/5a31313d94a949e5947594f484e6c68f is 50, key is test_row_0/A:col10/1731565034373/Put/seqid=0 2024-11-14T06:17:14,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742119_1295 (size=12301) 2024-11-14T06:17:14,419 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5be089a788a7af479f29a70b78ed76e8#B#compaction#250 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:17:14,419 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/0827ed6da7584469bf98962bf2ddb17f is 50, key is test_row_0/B:col10/1731565034299/Put/seqid=0 2024-11-14T06:17:14,420 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5be089a788a7af479f29a70b78ed76e8#A#compaction#251 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:17:14,421 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/52b7dcb81810469a90ec6b7a42cabcc1 is 50, key is test_row_0/A:col10/1731565034299/Put/seqid=0 2024-11-14T06:17:14,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742121_1297 (size=12765) 2024-11-14T06:17:14,431 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/52b7dcb81810469a90ec6b7a42cabcc1 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/52b7dcb81810469a90ec6b7a42cabcc1 2024-11-14T06:17:14,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742120_1296 (size=12765) 2024-11-14T06:17:14,437 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5be089a788a7af479f29a70b78ed76e8/A of 5be089a788a7af479f29a70b78ed76e8 into 52b7dcb81810469a90ec6b7a42cabcc1(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:17:14,437 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5be089a788a7af479f29a70b78ed76e8: 2024-11-14T06:17:14,437 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., storeName=5be089a788a7af479f29a70b78ed76e8/A, priority=13, startTime=1731565034403; duration=0sec 2024-11-14T06:17:14,437 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:17:14,437 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5be089a788a7af479f29a70b78ed76e8:A 2024-11-14T06:17:14,437 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:17:14,440 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:17:14,440 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): 5be089a788a7af479f29a70b78ed76e8/C is initiating minor compaction (all files) 2024-11-14T06:17:14,440 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5be089a788a7af479f29a70b78ed76e8/C in TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:14,440 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/b39c10af3b7b4786a72c00ce337c31ec, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/6c0388b8085c45d48008b46d44d9c432, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/9e492613c9a14aa6ae215a1461804e34] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp, totalSize=36.1 K 2024-11-14T06:17:14,440 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting b39c10af3b7b4786a72c00ce337c31ec, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731565030931 2024-11-14T06:17:14,441 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c0388b8085c45d48008b46d44d9c432, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731565032016 2024-11-14T06:17:14,441 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e492613c9a14aa6ae215a1461804e34, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731565034187 2024-11-14T06:17:14,448 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5be089a788a7af479f29a70b78ed76e8#C#compaction#252 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:17:14,449 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/fa9fac7eae584be3bc715211d9db5ba4 is 50, key is test_row_0/C:col10/1731565034299/Put/seqid=0 2024-11-14T06:17:14,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742122_1298 (size=12765) 2024-11-14T06:17:14,458 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/fa9fac7eae584be3bc715211d9db5ba4 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/fa9fac7eae584be3bc715211d9db5ba4 2024-11-14T06:17:14,463 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5be089a788a7af479f29a70b78ed76e8/C of 5be089a788a7af479f29a70b78ed76e8 into fa9fac7eae584be3bc715211d9db5ba4(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:17:14,463 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5be089a788a7af479f29a70b78ed76e8: 2024-11-14T06:17:14,463 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., storeName=5be089a788a7af479f29a70b78ed76e8/C, priority=13, startTime=1731565034403; duration=0sec 2024-11-14T06:17:14,464 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:14,464 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5be089a788a7af479f29a70b78ed76e8:C 2024-11-14T06:17:14,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:14,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33182 deadline: 1731565094461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:14,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:14,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:14,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:14,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:14,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33126 deadline: 1731565094465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:14,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:14,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:14,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:14,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:14,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33182 deadline: 1731565094577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:14,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:14,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:14,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33126 deadline: 1731565094577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:14,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:14,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:14,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:14,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:14,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:14,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:14,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33182 deadline: 1731565094787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:14,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:14,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:14,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33126 deadline: 1731565094787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:14,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:14,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:14,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:14,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:14,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:14,819 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/5a31313d94a949e5947594f484e6c68f 2024-11-14T06:17:14,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/b283850f557f43ce977c785b638f55c8 is 50, key is test_row_0/B:col10/1731565034373/Put/seqid=0 2024-11-14T06:17:14,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742123_1299 (size=12301) 2024-11-14T06:17:14,840 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/0827ed6da7584469bf98962bf2ddb17f as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/0827ed6da7584469bf98962bf2ddb17f 2024-11-14T06:17:14,847 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5be089a788a7af479f29a70b78ed76e8/B of 5be089a788a7af479f29a70b78ed76e8 into 0827ed6da7584469bf98962bf2ddb17f(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:17:14,847 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5be089a788a7af479f29a70b78ed76e8: 2024-11-14T06:17:14,847 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., storeName=5be089a788a7af479f29a70b78ed76e8/B, priority=13, startTime=1731565034403; duration=0sec 2024-11-14T06:17:14,847 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:14,847 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5be089a788a7af479f29a70b78ed76e8:B 2024-11-14T06:17:15,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:15,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33126 deadline: 1731565095097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:15,100 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:15,100 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:15,101 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:15,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:15,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33182 deadline: 1731565095100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:15,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:15,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:15,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:15,231 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.68 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/b283850f557f43ce977c785b638f55c8 2024-11-14T06:17:15,255 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/0fc8bf4fbbad49df970bb7a3dff049e2 is 50, key is test_row_0/C:col10/1731565034373/Put/seqid=0 2024-11-14T06:17:15,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742124_1300 (size=12301) 2024-11-14T06:17:15,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:15,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33126 deadline: 1731565095608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:15,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:15,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:15,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:15,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:15,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33182 deadline: 1731565095617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:15,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:15,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:15,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:15,670 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.68 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/0fc8bf4fbbad49df970bb7a3dff049e2 2024-11-14T06:17:15,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/5a31313d94a949e5947594f484e6c68f as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/5a31313d94a949e5947594f484e6c68f 2024-11-14T06:17:15,680 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/5a31313d94a949e5947594f484e6c68f, entries=150, sequenceid=279, filesize=12.0 K 2024-11-14T06:17:15,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/b283850f557f43ce977c785b638f55c8 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/b283850f557f43ce977c785b638f55c8 2024-11-14T06:17:15,686 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/b283850f557f43ce977c785b638f55c8, entries=150, sequenceid=279, filesize=12.0 K 2024-11-14T06:17:15,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/0fc8bf4fbbad49df970bb7a3dff049e2 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/0fc8bf4fbbad49df970bb7a3dff049e2 2024-11-14T06:17:15,703 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/0fc8bf4fbbad49df970bb7a3dff049e2, entries=150, sequenceid=279, filesize=12.0 K 2024-11-14T06:17:15,704 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~88.66 KB/90792, heapSize ~248.73 KB/254704, currentSize=93.93 KB/96180 for 5be089a788a7af479f29a70b78ed76e8 in 1301ms, sequenceid=279, compaction requested=false 2024-11-14T06:17:15,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5be089a788a7af479f29a70b78ed76e8: 2024-11-14T06:17:16,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-14T06:17:16,087 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:17:16,088 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T06:17:16,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-14T06:17:16,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T06:17:16,090 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T06:17:16,090 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T06:17:16,091 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T06:17:16,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T06:17:16,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on 5be089a788a7af479f29a70b78ed76e8 2024-11-14T06:17:16,210 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5be089a788a7af479f29a70b78ed76e8 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-14T06:17:16,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=A 2024-11-14T06:17:16,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:16,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=B 2024-11-14T06:17:16,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:16,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=C 2024-11-14T06:17:16,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:16,216 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/4bf47f7894b64ffb800fd328b4d7d689 is 50, key is test_row_0/A:col10/1731565034459/Put/seqid=0 2024-11-14T06:17:16,243 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-14T06:17:16,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:16,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:16,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:16,244 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:16,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:16,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:16,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742125_1301 (size=14741) 2024-11-14T06:17:16,317 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/4bf47f7894b64ffb800fd328b4d7d689 2024-11-14T06:17:16,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:16,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33098 deadline: 1731565096318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:16,324 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:16,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:16,324 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:16,324 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:16,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33100 deadline: 1731565096318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:16,325 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:16,325 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:16,325 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:16,328 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/b54c4b265c00443090d76ed29aaef75e is 50, key is test_row_0/B:col10/1731565034459/Put/seqid=0 2024-11-14T06:17:16,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742126_1302 (size=12301) 2024-11-14T06:17:16,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-14T06:17:16,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:16,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:16,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:16,397 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:16,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:16,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:16,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T06:17:16,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:16,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33098 deadline: 1731565096429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:16,433 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:16,433 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:16,433 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:16,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:16,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33100 deadline: 1731565096432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:16,436 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:16,436 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:16,436 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:16,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-14T06:17:16,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:16,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:16,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:16,560 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:16,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:16,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:16,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:16,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33126 deadline: 1731565096617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:16,622 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to ef93ef3a83c5,42283,1731564965945 for 'test_row_2' in 5be089a788a7af479f29a70b78ed76e8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T06:17:16,622 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:16,622 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:16,622 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:16,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:16,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33182 deadline: 1731565096627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:16,630 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to ef93ef3a83c5,42283,1731564965945 for 'test_row_0' in 5be089a788a7af479f29a70b78ed76e8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T06:17:16,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:16,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:16,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:16,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:16,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33098 deadline: 1731565096637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:16,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:16,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:16,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:16,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:16,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33100 deadline: 1731565096647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:16,651 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:16,651 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:16,651 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:16,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-14T06:17:16,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:16,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:16,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:16,716 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:16,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:16,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:16,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T06:17:16,750 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/b54c4b265c00443090d76ed29aaef75e 2024-11-14T06:17:16,762 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/918da577bb91456693c8dacecbbc6847 is 50, key is test_row_0/C:col10/1731565034459/Put/seqid=0 2024-11-14T06:17:16,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742127_1303 (size=12301) 2024-11-14T06:17:16,797 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/918da577bb91456693c8dacecbbc6847 2024-11-14T06:17:16,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/4bf47f7894b64ffb800fd328b4d7d689 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/4bf47f7894b64ffb800fd328b4d7d689 2024-11-14T06:17:16,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/4bf47f7894b64ffb800fd328b4d7d689, entries=200, sequenceid=300, filesize=14.4 K 2024-11-14T06:17:16,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/b54c4b265c00443090d76ed29aaef75e as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/b54c4b265c00443090d76ed29aaef75e 2024-11-14T06:17:16,822 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/b54c4b265c00443090d76ed29aaef75e, entries=150, sequenceid=300, filesize=12.0 K 2024-11-14T06:17:16,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/918da577bb91456693c8dacecbbc6847 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/918da577bb91456693c8dacecbbc6847 2024-11-14T06:17:16,834 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/918da577bb91456693c8dacecbbc6847, entries=150, sequenceid=300, filesize=12.0 K 2024-11-14T06:17:16,835 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 5be089a788a7af479f29a70b78ed76e8 in 625ms, sequenceid=300, compaction requested=true 2024-11-14T06:17:16,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5be089a788a7af479f29a70b78ed76e8: 2024-11-14T06:17:16,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5be089a788a7af479f29a70b78ed76e8:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T06:17:16,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:16,836 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:17:16,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5be089a788a7af479f29a70b78ed76e8:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T06:17:16,836 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:17:16,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:16,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5be089a788a7af479f29a70b78ed76e8:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T06:17:16,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:17:16,837 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39807 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:17:16,837 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:17:16,837 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): 5be089a788a7af479f29a70b78ed76e8/A is initiating minor compaction (all files) 2024-11-14T06:17:16,837 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): 5be089a788a7af479f29a70b78ed76e8/B is initiating minor compaction (all files) 2024-11-14T06:17:16,838 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5be089a788a7af479f29a70b78ed76e8/B in TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:16,838 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5be089a788a7af479f29a70b78ed76e8/A in TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:16,838 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/52b7dcb81810469a90ec6b7a42cabcc1, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/5a31313d94a949e5947594f484e6c68f, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/4bf47f7894b64ffb800fd328b4d7d689] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp, totalSize=38.9 K 2024-11-14T06:17:16,838 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/0827ed6da7584469bf98962bf2ddb17f, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/b283850f557f43ce977c785b638f55c8, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/b54c4b265c00443090d76ed29aaef75e] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp, totalSize=36.5 K 2024-11-14T06:17:16,838 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 0827ed6da7584469bf98962bf2ddb17f, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731565034187 2024-11-14T06:17:16,838 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52b7dcb81810469a90ec6b7a42cabcc1, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731565034187 2024-11-14T06:17:16,838 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a31313d94a949e5947594f484e6c68f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731565034373 2024-11-14T06:17:16,838 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting b283850f557f43ce977c785b638f55c8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731565034373 2024-11-14T06:17:16,839 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4bf47f7894b64ffb800fd328b4d7d689, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731565034453 2024-11-14T06:17:16,839 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting b54c4b265c00443090d76ed29aaef75e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731565034459 2024-11-14T06:17:16,850 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5be089a788a7af479f29a70b78ed76e8#A#compaction#258 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:17:16,851 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/47e6aa723f4b4e018092caa86ef47b66 is 50, key is test_row_0/A:col10/1731565034459/Put/seqid=0 2024-11-14T06:17:16,853 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5be089a788a7af479f29a70b78ed76e8#B#compaction#259 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:17:16,853 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/66f88786085e41d1bf4908e5dbbdfd9a is 50, key is test_row_0/B:col10/1731565034459/Put/seqid=0 2024-11-14T06:17:16,869 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-14T06:17:16,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:16,869 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 5be089a788a7af479f29a70b78ed76e8 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-14T06:17:16,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=A 2024-11-14T06:17:16,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:16,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=B 2024-11-14T06:17:16,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:16,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=C 2024-11-14T06:17:16,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:16,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/92d9da2b91e94d9eaf0cfbf51fbf599f is 50, key is test_row_0/A:col10/1731565036251/Put/seqid=0 2024-11-14T06:17:16,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742128_1304 (size=13017) 2024-11-14T06:17:16,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742129_1305 (size=13017) 2024-11-14T06:17:16,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742130_1306 (size=12301) 2024-11-14T06:17:16,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on 5be089a788a7af479f29a70b78ed76e8 2024-11-14T06:17:16,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:16,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:16,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33098 deadline: 1731565096988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:16,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:16,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:16,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:16,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:16,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33100 deadline: 1731565096993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:16,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:16,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:16,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:17,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:17,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33098 deadline: 1731565097097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:17,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:17,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:17,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:17,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:17,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33100 deadline: 1731565097107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:17,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:17,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:17,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:17,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T06:17:17,290 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/47e6aa723f4b4e018092caa86ef47b66 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/47e6aa723f4b4e018092caa86ef47b66 2024-11-14T06:17:17,304 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/92d9da2b91e94d9eaf0cfbf51fbf599f 2024-11-14T06:17:17,304 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5be089a788a7af479f29a70b78ed76e8/A of 5be089a788a7af479f29a70b78ed76e8 into 47e6aa723f4b4e018092caa86ef47b66(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:17:17,304 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5be089a788a7af479f29a70b78ed76e8: 2024-11-14T06:17:17,304 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., storeName=5be089a788a7af479f29a70b78ed76e8/A, priority=13, startTime=1731565036836; duration=0sec 2024-11-14T06:17:17,304 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:17:17,304 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5be089a788a7af479f29a70b78ed76e8:A 2024-11-14T06:17:17,304 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:17:17,305 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/66f88786085e41d1bf4908e5dbbdfd9a as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/66f88786085e41d1bf4908e5dbbdfd9a 2024-11-14T06:17:17,306 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:17:17,307 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): 5be089a788a7af479f29a70b78ed76e8/C is initiating minor compaction (all files) 2024-11-14T06:17:17,307 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5be089a788a7af479f29a70b78ed76e8/C in TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:17,307 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/fa9fac7eae584be3bc715211d9db5ba4, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/0fc8bf4fbbad49df970bb7a3dff049e2, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/918da577bb91456693c8dacecbbc6847] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp, totalSize=36.5 K 2024-11-14T06:17:17,307 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa9fac7eae584be3bc715211d9db5ba4, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731565034187 2024-11-14T06:17:17,308 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0fc8bf4fbbad49df970bb7a3dff049e2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731565034373 2024-11-14T06:17:17,308 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 918da577bb91456693c8dacecbbc6847, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731565034459 2024-11-14T06:17:17,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:17,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33098 deadline: 1731565097308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:17,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:17,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:17,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:17,311 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5be089a788a7af479f29a70b78ed76e8/B of 5be089a788a7af479f29a70b78ed76e8 into 66f88786085e41d1bf4908e5dbbdfd9a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:17:17,311 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5be089a788a7af479f29a70b78ed76e8: 2024-11-14T06:17:17,311 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., storeName=5be089a788a7af479f29a70b78ed76e8/B, priority=13, startTime=1731565036836; duration=0sec 2024-11-14T06:17:17,311 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:17,311 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5be089a788a7af479f29a70b78ed76e8:B 2024-11-14T06:17:17,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/6ca4b758b36c49fe848bfe8975ec91bc is 50, key is test_row_0/B:col10/1731565036251/Put/seqid=0 2024-11-14T06:17:17,316 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5be089a788a7af479f29a70b78ed76e8#C#compaction#262 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:17:17,316 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/ea0ad650abca4dd6a4b98d0cb21e3c0a is 50, key is test_row_0/C:col10/1731565034459/Put/seqid=0 2024-11-14T06:17:17,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742131_1307 (size=12301) 2024-11-14T06:17:17,319 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/6ca4b758b36c49fe848bfe8975ec91bc 2024-11-14T06:17:17,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742132_1308 (size=13017) 2024-11-14T06:17:17,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:17,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33100 deadline: 1731565097324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:17,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:17,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:17,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:17,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/ad93dfc3092645729d76bab87ba5bdef is 50, key is test_row_0/C:col10/1731565036251/Put/seqid=0 2024-11-14T06:17:17,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742133_1309 (size=12301) 2024-11-14T06:17:17,341 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/ad93dfc3092645729d76bab87ba5bdef 2024-11-14T06:17:17,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/92d9da2b91e94d9eaf0cfbf51fbf599f as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/92d9da2b91e94d9eaf0cfbf51fbf599f 2024-11-14T06:17:17,353 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/92d9da2b91e94d9eaf0cfbf51fbf599f, entries=150, sequenceid=315, filesize=12.0 K 2024-11-14T06:17:17,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/6ca4b758b36c49fe848bfe8975ec91bc as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/6ca4b758b36c49fe848bfe8975ec91bc 2024-11-14T06:17:17,358 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/6ca4b758b36c49fe848bfe8975ec91bc, entries=150, sequenceid=315, filesize=12.0 K 2024-11-14T06:17:17,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/ad93dfc3092645729d76bab87ba5bdef as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/ad93dfc3092645729d76bab87ba5bdef 2024-11-14T06:17:17,364 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/ad93dfc3092645729d76bab87ba5bdef, entries=150, sequenceid=315, filesize=12.0 K 2024-11-14T06:17:17,365 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 5be089a788a7af479f29a70b78ed76e8 in 496ms, sequenceid=315, compaction requested=false 2024-11-14T06:17:17,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 5be089a788a7af479f29a70b78ed76e8: 2024-11-14T06:17:17,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:17,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-14T06:17:17,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-14T06:17:17,369 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-14T06:17:17,369 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2760 sec 2024-11-14T06:17:17,371 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.2810 sec 2024-11-14T06:17:17,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on 5be089a788a7af479f29a70b78ed76e8 2024-11-14T06:17:17,621 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5be089a788a7af479f29a70b78ed76e8 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-14T06:17:17,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=A 2024-11-14T06:17:17,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:17,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=B 2024-11-14T06:17:17,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:17,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=C 2024-11-14T06:17:17,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:17,626 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/2cbcfa7dceae4608b8ad3e11b84324fd is 50, key is test_row_0/A:col10/1731565037619/Put/seqid=0 2024-11-14T06:17:17,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742134_1310 (size=14741) 2024-11-14T06:17:17,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:17,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33098 deadline: 1731565097656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:17,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:17,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:17,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:17,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:17,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33100 deadline: 1731565097662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:17,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:17,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:17,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:17,736 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/ea0ad650abca4dd6a4b98d0cb21e3c0a as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/ea0ad650abca4dd6a4b98d0cb21e3c0a 2024-11-14T06:17:17,748 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5be089a788a7af479f29a70b78ed76e8/C of 5be089a788a7af479f29a70b78ed76e8 into ea0ad650abca4dd6a4b98d0cb21e3c0a(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:17:17,748 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5be089a788a7af479f29a70b78ed76e8: 2024-11-14T06:17:17,748 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., storeName=5be089a788a7af479f29a70b78ed76e8/C, priority=13, startTime=1731565036836; duration=0sec 2024-11-14T06:17:17,749 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:17,749 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5be089a788a7af479f29a70b78ed76e8:C 2024-11-14T06:17:17,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:17,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33098 deadline: 1731565097767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:17,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:17,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:17,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:17,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:17,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33100 deadline: 1731565097777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:17,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:17,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:17,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:17,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:17,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33098 deadline: 1731565097977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:17,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:17,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:17,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:17,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:17,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33100 deadline: 1731565097987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:17,991 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:17,991 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:17,991 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:18,033 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/2cbcfa7dceae4608b8ad3e11b84324fd 2024-11-14T06:17:18,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/d0fd7aa1486d478d8f8a9eea5f97dcc3 is 50, key is test_row_0/B:col10/1731565037619/Put/seqid=0 2024-11-14T06:17:18,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742135_1311 (size=12301) 2024-11-14T06:17:18,051 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/d0fd7aa1486d478d8f8a9eea5f97dcc3 2024-11-14T06:17:18,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/97b063eb7e7a40ac94d2e04f5b9a9049 is 50, key is test_row_0/C:col10/1731565037619/Put/seqid=0 2024-11-14T06:17:18,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742136_1312 (size=12301) 2024-11-14T06:17:18,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T06:17:18,237 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:17:18,237 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T06:17:18,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-14T06:17:18,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-14T06:17:18,239 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T06:17:18,240 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T06:17:18,240 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T06:17:18,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:18,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33098 deadline: 1731565098287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:18,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:18,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:18,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:18,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:18,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33100 deadline: 1731565098297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:18,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:18,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:18,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:18,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-14T06:17:18,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T06:17:18,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:18,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:18,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:18,392 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:18,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:18,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:18,470 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/97b063eb7e7a40ac94d2e04f5b9a9049 2024-11-14T06:17:18,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/2cbcfa7dceae4608b8ad3e11b84324fd as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/2cbcfa7dceae4608b8ad3e11b84324fd 2024-11-14T06:17:18,497 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/2cbcfa7dceae4608b8ad3e11b84324fd, entries=200, sequenceid=335, filesize=14.4 K 2024-11-14T06:17:18,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/d0fd7aa1486d478d8f8a9eea5f97dcc3 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/d0fd7aa1486d478d8f8a9eea5f97dcc3 2024-11-14T06:17:18,502 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/d0fd7aa1486d478d8f8a9eea5f97dcc3, entries=150, sequenceid=335, filesize=12.0 K 2024-11-14T06:17:18,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/97b063eb7e7a40ac94d2e04f5b9a9049 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/97b063eb7e7a40ac94d2e04f5b9a9049 2024-11-14T06:17:18,510 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/97b063eb7e7a40ac94d2e04f5b9a9049, entries=150, sequenceid=335, filesize=12.0 K 2024-11-14T06:17:18,512 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 5be089a788a7af479f29a70b78ed76e8 in 890ms, sequenceid=335, compaction requested=true 2024-11-14T06:17:18,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5be089a788a7af479f29a70b78ed76e8: 2024-11-14T06:17:18,512 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:17:18,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5be089a788a7af479f29a70b78ed76e8:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T06:17:18,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:18,512 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:17:18,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5be089a788a7af479f29a70b78ed76e8:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T06:17:18,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:18,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5be089a788a7af479f29a70b78ed76e8:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T06:17:18,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:17:18,513 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:17:18,513 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): 5be089a788a7af479f29a70b78ed76e8/B is initiating minor compaction (all files) 2024-11-14T06:17:18,513 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5be089a788a7af479f29a70b78ed76e8/B in TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:18,514 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/66f88786085e41d1bf4908e5dbbdfd9a, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/6ca4b758b36c49fe848bfe8975ec91bc, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/d0fd7aa1486d478d8f8a9eea5f97dcc3] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp, totalSize=36.7 K 2024-11-14T06:17:18,514 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40059 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:17:18,514 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): 5be089a788a7af479f29a70b78ed76e8/A is initiating minor compaction (all files) 2024-11-14T06:17:18,514 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5be089a788a7af479f29a70b78ed76e8/A in TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:18,514 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/47e6aa723f4b4e018092caa86ef47b66, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/92d9da2b91e94d9eaf0cfbf51fbf599f, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/2cbcfa7dceae4608b8ad3e11b84324fd] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp, totalSize=39.1 K 2024-11-14T06:17:18,514 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 66f88786085e41d1bf4908e5dbbdfd9a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731565034459 2024-11-14T06:17:18,514 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47e6aa723f4b4e018092caa86ef47b66, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731565034459 2024-11-14T06:17:18,515 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ca4b758b36c49fe848bfe8975ec91bc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731565036235 2024-11-14T06:17:18,515 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92d9da2b91e94d9eaf0cfbf51fbf599f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731565036235 2024-11-14T06:17:18,515 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting d0fd7aa1486d478d8f8a9eea5f97dcc3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731565036979 2024-11-14T06:17:18,515 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2cbcfa7dceae4608b8ad3e11b84324fd, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731565036979 2024-11-14T06:17:18,528 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5be089a788a7af479f29a70b78ed76e8#A#compaction#267 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:17:18,529 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/f999f07d433d40919f25da21e11b9c2d is 50, key is test_row_0/A:col10/1731565037619/Put/seqid=0 2024-11-14T06:17:18,538 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5be089a788a7af479f29a70b78ed76e8#B#compaction#268 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:17:18,540 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/b1cf24a4c1d0489e96ceba77a4b0e54c is 50, key is test_row_0/B:col10/1731565037619/Put/seqid=0 2024-11-14T06:17:18,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T06:17:18,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:18,546 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 5be089a788a7af479f29a70b78ed76e8 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-14T06:17:18,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=A 2024-11-14T06:17:18,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:18,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=B 2024-11-14T06:17:18,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:18,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=C 2024-11-14T06:17:18,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:18,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-14T06:17:18,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/e37ffec24d4c43d8b3937724e99c1aa2 is 50, key is test_row_0/A:col10/1731565037653/Put/seqid=0 2024-11-14T06:17:18,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742137_1313 (size=13119) 2024-11-14T06:17:18,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742138_1314 (size=13119) 2024-11-14T06:17:18,576 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/b1cf24a4c1d0489e96ceba77a4b0e54c as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/b1cf24a4c1d0489e96ceba77a4b0e54c 2024-11-14T06:17:18,588 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5be089a788a7af479f29a70b78ed76e8/B of 5be089a788a7af479f29a70b78ed76e8 into b1cf24a4c1d0489e96ceba77a4b0e54c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:17:18,588 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5be089a788a7af479f29a70b78ed76e8: 2024-11-14T06:17:18,588 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., storeName=5be089a788a7af479f29a70b78ed76e8/B, priority=13, startTime=1731565038512; duration=0sec 2024-11-14T06:17:18,589 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:17:18,589 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5be089a788a7af479f29a70b78ed76e8:B 2024-11-14T06:17:18,589 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:17:18,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742139_1315 (size=12301) 2024-11-14T06:17:18,590 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/e37ffec24d4c43d8b3937724e99c1aa2 2024-11-14T06:17:18,592 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/f999f07d433d40919f25da21e11b9c2d as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/f999f07d433d40919f25da21e11b9c2d 2024-11-14T06:17:18,598 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:17:18,599 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): 5be089a788a7af479f29a70b78ed76e8/C is initiating minor compaction (all files) 2024-11-14T06:17:18,604 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5be089a788a7af479f29a70b78ed76e8/C in TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:18,604 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/ea0ad650abca4dd6a4b98d0cb21e3c0a, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/ad93dfc3092645729d76bab87ba5bdef, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/97b063eb7e7a40ac94d2e04f5b9a9049] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp, totalSize=36.7 K 2024-11-14T06:17:18,605 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting ea0ad650abca4dd6a4b98d0cb21e3c0a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731565034459 2024-11-14T06:17:18,606 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting ad93dfc3092645729d76bab87ba5bdef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731565036235 2024-11-14T06:17:18,608 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 97b063eb7e7a40ac94d2e04f5b9a9049, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731565036979 2024-11-14T06:17:18,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/246ddb2498964d3692ac0bd58dbdf92c is 50, key is test_row_0/B:col10/1731565037653/Put/seqid=0 2024-11-14T06:17:18,611 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5be089a788a7af479f29a70b78ed76e8/A of 5be089a788a7af479f29a70b78ed76e8 into f999f07d433d40919f25da21e11b9c2d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:17:18,611 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5be089a788a7af479f29a70b78ed76e8: 2024-11-14T06:17:18,611 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., storeName=5be089a788a7af479f29a70b78ed76e8/A, priority=13, startTime=1731565038512; duration=0sec 2024-11-14T06:17:18,611 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:18,611 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5be089a788a7af479f29a70b78ed76e8:A 2024-11-14T06:17:18,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742140_1316 (size=12301) 2024-11-14T06:17:18,635 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/246ddb2498964d3692ac0bd58dbdf92c 2024-11-14T06:17:18,645 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5be089a788a7af479f29a70b78ed76e8#C#compaction#271 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:17:18,645 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/ecf37bf7b61242ffbff04ae6784a242e is 50, key is test_row_0/C:col10/1731565037619/Put/seqid=0 2024-11-14T06:17:18,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on 5be089a788a7af479f29a70b78ed76e8 2024-11-14T06:17:18,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:18,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/6067b163357e4a97a3837eb26050bf9d is 50, key is test_row_0/C:col10/1731565037653/Put/seqid=0 2024-11-14T06:17:18,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742141_1317 (size=13119) 2024-11-14T06:17:18,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742142_1318 (size=12301) 2024-11-14T06:17:18,696 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/6067b163357e4a97a3837eb26050bf9d 2024-11-14T06:17:18,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:18,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33182 deadline: 1731565098690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:18,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:18,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:18,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:18,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:18,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33126 deadline: 1731565098699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:18,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:18,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:18,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:18,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/e37ffec24d4c43d8b3937724e99c1aa2 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/e37ffec24d4c43d8b3937724e99c1aa2 2024-11-14T06:17:18,711 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/e37ffec24d4c43d8b3937724e99c1aa2, entries=150, sequenceid=350, filesize=12.0 K 2024-11-14T06:17:18,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/246ddb2498964d3692ac0bd58dbdf92c as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/246ddb2498964d3692ac0bd58dbdf92c 2024-11-14T06:17:18,727 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/246ddb2498964d3692ac0bd58dbdf92c, entries=150, sequenceid=350, filesize=12.0 K 2024-11-14T06:17:18,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/6067b163357e4a97a3837eb26050bf9d as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/6067b163357e4a97a3837eb26050bf9d 2024-11-14T06:17:18,737 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/6067b163357e4a97a3837eb26050bf9d, entries=150, sequenceid=350, filesize=12.0 K 2024-11-14T06:17:18,738 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 5be089a788a7af479f29a70b78ed76e8 in 192ms, sequenceid=350, compaction requested=false 2024-11-14T06:17:18,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 5be089a788a7af479f29a70b78ed76e8: 2024-11-14T06:17:18,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:18,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-14T06:17:18,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-14T06:17:18,743 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-14T06:17:18,743 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 500 msec 2024-11-14T06:17:18,746 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 506 msec 2024-11-14T06:17:18,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on 5be089a788a7af479f29a70b78ed76e8 2024-11-14T06:17:18,800 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5be089a788a7af479f29a70b78ed76e8 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-14T06:17:18,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=A 2024-11-14T06:17:18,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:18,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=B 2024-11-14T06:17:18,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:18,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=C 2024-11-14T06:17:18,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:18,814 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/f6b3c386ecf94e819d733d92480ec646 is 50, key is test_row_0/A:col10/1731565038696/Put/seqid=0 2024-11-14T06:17:18,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:18,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33126 deadline: 1731565098824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:18,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:18,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33098 deadline: 1731565098826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:18,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:18,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33100 deadline: 1731565098823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:18,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:18,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33182 deadline: 1731565098826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:18,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:18,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:18,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:18,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:18,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:18,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:18,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:18,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:18,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:18,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:18,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:18,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:18,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742143_1319 (size=14741) 2024-11-14T06:17:18,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-14T06:17:18,873 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:17:18,873 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T06:17:18,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-14T06:17:18,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-14T06:17:18,875 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T06:17:18,877 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T06:17:18,877 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T06:17:18,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:18,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33098 deadline: 1731565098938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:18,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:18,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:18,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:18,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:18,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33182 deadline: 1731565098938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:18,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:18,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:18,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:18,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:18,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33100 deadline: 1731565098938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:18,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:18,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:18,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:18,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:18,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33126 deadline: 1731565098938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:18,941 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:18,941 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:18,941 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:18,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-14T06:17:19,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-14T06:17:19,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:19,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:19,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:19,030 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:19,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:19,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:19,087 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/ecf37bf7b61242ffbff04ae6784a242e as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/ecf37bf7b61242ffbff04ae6784a242e 2024-11-14T06:17:19,098 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5be089a788a7af479f29a70b78ed76e8/C of 5be089a788a7af479f29a70b78ed76e8 into ecf37bf7b61242ffbff04ae6784a242e(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:17:19,098 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5be089a788a7af479f29a70b78ed76e8: 2024-11-14T06:17:19,098 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., storeName=5be089a788a7af479f29a70b78ed76e8/C, priority=13, startTime=1731565038513; duration=0sec 2024-11-14T06:17:19,098 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:19,098 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5be089a788a7af479f29a70b78ed76e8:C 2024-11-14T06:17:19,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:19,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33126 deadline: 1731565099147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:19,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:19,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:19,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:19,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33098 deadline: 1731565099148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:19,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:19,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:19,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33182 deadline: 1731565099148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:19,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:19,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:19,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:19,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:19,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33100 deadline: 1731565099148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:19,153 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:19,153 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:19,153 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:19,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:19,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:19,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:19,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-14T06:17:19,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:19,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:19,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:19,182 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:19,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:19,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:19,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-14T06:17:19,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/f6b3c386ecf94e819d733d92480ec646 2024-11-14T06:17:19,258 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/a73e56a7396041fb918e466de9b4509d is 50, key is test_row_0/B:col10/1731565038696/Put/seqid=0 2024-11-14T06:17:19,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742144_1320 (size=12301) 2024-11-14T06:17:19,274 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/a73e56a7396041fb918e466de9b4509d 2024-11-14T06:17:19,281 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/078bb9e0b5264e819e27ee8289b7f9b9 is 50, key is test_row_0/C:col10/1731565038696/Put/seqid=0 2024-11-14T06:17:19,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742145_1321 (size=12301) 2024-11-14T06:17:19,286 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/078bb9e0b5264e819e27ee8289b7f9b9 2024-11-14T06:17:19,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/f6b3c386ecf94e819d733d92480ec646 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/f6b3c386ecf94e819d733d92480ec646 2024-11-14T06:17:19,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/f6b3c386ecf94e819d733d92480ec646, entries=200, sequenceid=371, filesize=14.4 K 2024-11-14T06:17:19,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/a73e56a7396041fb918e466de9b4509d as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/a73e56a7396041fb918e466de9b4509d 2024-11-14T06:17:19,302 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/a73e56a7396041fb918e466de9b4509d, entries=150, sequenceid=371, filesize=12.0 K 2024-11-14T06:17:19,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/078bb9e0b5264e819e27ee8289b7f9b9 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/078bb9e0b5264e819e27ee8289b7f9b9 2024-11-14T06:17:19,306 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/078bb9e0b5264e819e27ee8289b7f9b9, entries=150, sequenceid=371, filesize=12.0 K 2024-11-14T06:17:19,309 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 5be089a788a7af479f29a70b78ed76e8 in 509ms, sequenceid=371, compaction requested=true 2024-11-14T06:17:19,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5be089a788a7af479f29a70b78ed76e8: 2024-11-14T06:17:19,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5be089a788a7af479f29a70b78ed76e8:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T06:17:19,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:19,309 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:17:19,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5be089a788a7af479f29a70b78ed76e8:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T06:17:19,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:19,309 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:17:19,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5be089a788a7af479f29a70b78ed76e8:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T06:17:19,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:17:19,317 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40161 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:17:19,317 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): 5be089a788a7af479f29a70b78ed76e8/A is initiating minor compaction (all files) 2024-11-14T06:17:19,317 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5be089a788a7af479f29a70b78ed76e8/A in TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:19,318 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/f999f07d433d40919f25da21e11b9c2d, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/e37ffec24d4c43d8b3937724e99c1aa2, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/f6b3c386ecf94e819d733d92480ec646] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp, totalSize=39.2 K 2024-11-14T06:17:19,320 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:17:19,320 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): 5be089a788a7af479f29a70b78ed76e8/B is initiating minor compaction (all files) 2024-11-14T06:17:19,320 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting f999f07d433d40919f25da21e11b9c2d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731565036979 2024-11-14T06:17:19,320 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5be089a788a7af479f29a70b78ed76e8/B in TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:19,320 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/b1cf24a4c1d0489e96ceba77a4b0e54c, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/246ddb2498964d3692ac0bd58dbdf92c, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/a73e56a7396041fb918e466de9b4509d] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp, totalSize=36.8 K 2024-11-14T06:17:19,320 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting e37ffec24d4c43d8b3937724e99c1aa2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1731565037647 2024-11-14T06:17:19,321 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting b1cf24a4c1d0489e96ceba77a4b0e54c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731565036979 2024-11-14T06:17:19,322 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6b3c386ecf94e819d733d92480ec646, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731565038675 2024-11-14T06:17:19,322 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 246ddb2498964d3692ac0bd58dbdf92c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1731565037647 2024-11-14T06:17:19,323 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting a73e56a7396041fb918e466de9b4509d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731565038675 2024-11-14T06:17:19,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-14T06:17:19,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:19,335 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 5be089a788a7af479f29a70b78ed76e8 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-14T06:17:19,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=A 2024-11-14T06:17:19,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:19,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=B 2024-11-14T06:17:19,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:19,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=C 2024-11-14T06:17:19,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:19,346 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5be089a788a7af479f29a70b78ed76e8#B#compaction#276 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:17:19,347 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/237e924538684d0ebb83314887a34d98 is 50, key is test_row_0/B:col10/1731565038696/Put/seqid=0 2024-11-14T06:17:19,354 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5be089a788a7af479f29a70b78ed76e8#A#compaction#277 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:17:19,355 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/f2783c99e62240a8bc21b728897811ec is 50, key is test_row_0/A:col10/1731565038696/Put/seqid=0 2024-11-14T06:17:19,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/8a402b69bece4ae9a8dc54e29751d806 is 50, key is test_row_0/A:col10/1731565038823/Put/seqid=0 2024-11-14T06:17:19,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742148_1324 (size=12301) 2024-11-14T06:17:19,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742147_1323 (size=13221) 2024-11-14T06:17:19,419 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/f2783c99e62240a8bc21b728897811ec as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/f2783c99e62240a8bc21b728897811ec 2024-11-14T06:17:19,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742146_1322 (size=13221) 2024-11-14T06:17:19,427 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5be089a788a7af479f29a70b78ed76e8/A of 5be089a788a7af479f29a70b78ed76e8 into f2783c99e62240a8bc21b728897811ec(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:17:19,427 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5be089a788a7af479f29a70b78ed76e8: 2024-11-14T06:17:19,427 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., storeName=5be089a788a7af479f29a70b78ed76e8/A, priority=13, startTime=1731565039309; duration=0sec 2024-11-14T06:17:19,427 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:17:19,427 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5be089a788a7af479f29a70b78ed76e8:A 2024-11-14T06:17:19,428 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:17:19,430 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:17:19,430 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): 5be089a788a7af479f29a70b78ed76e8/C is initiating minor compaction (all files) 2024-11-14T06:17:19,430 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5be089a788a7af479f29a70b78ed76e8/C in TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:19,430 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/ecf37bf7b61242ffbff04ae6784a242e, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/6067b163357e4a97a3837eb26050bf9d, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/078bb9e0b5264e819e27ee8289b7f9b9] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp, totalSize=36.8 K 2024-11-14T06:17:19,430 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting ecf37bf7b61242ffbff04ae6784a242e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731565036979 2024-11-14T06:17:19,431 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6067b163357e4a97a3837eb26050bf9d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1731565037647 2024-11-14T06:17:19,431 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 078bb9e0b5264e819e27ee8289b7f9b9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731565038675 2024-11-14T06:17:19,445 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5be089a788a7af479f29a70b78ed76e8#C#compaction#279 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:17:19,446 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/f15d29215aa94445890cd47ebfb53811 is 50, key is test_row_0/C:col10/1731565038696/Put/seqid=0 2024-11-14T06:17:19,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on 5be089a788a7af479f29a70b78ed76e8 2024-11-14T06:17:19,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:19,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742149_1325 (size=13221) 2024-11-14T06:17:19,469 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/f15d29215aa94445890cd47ebfb53811 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/f15d29215aa94445890cd47ebfb53811 2024-11-14T06:17:19,478 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5be089a788a7af479f29a70b78ed76e8/C of 5be089a788a7af479f29a70b78ed76e8 into f15d29215aa94445890cd47ebfb53811(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:17:19,478 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5be089a788a7af479f29a70b78ed76e8: 2024-11-14T06:17:19,478 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., storeName=5be089a788a7af479f29a70b78ed76e8/C, priority=13, startTime=1731565039309; duration=0sec 2024-11-14T06:17:19,478 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:19,478 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5be089a788a7af479f29a70b78ed76e8:C 2024-11-14T06:17:19,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:19,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33126 deadline: 1731565099482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:19,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:19,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:19,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:19,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:19,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33098 deadline: 1731565099484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:19,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:19,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:19,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:19,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:19,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33182 deadline: 1731565099487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:19,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:19,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:19,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:19,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:19,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33100 deadline: 1731565099490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:19,496 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:19,496 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:19,496 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:19,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-14T06:17:19,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:19,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33098 deadline: 1731565099597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:19,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:19,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:19,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:19,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:19,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33126 deadline: 1731565099597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:19,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:19,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:19,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:19,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:19,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33182 deadline: 1731565099597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:19,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:19,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:19,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:19,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:19,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33100 deadline: 1731565099607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:19,608 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:19,608 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:19,608 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:19,799 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/8a402b69bece4ae9a8dc54e29751d806 2024-11-14T06:17:19,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/02938b666cbd45e5975b805ac84e868f is 50, key is test_row_0/B:col10/1731565038823/Put/seqid=0 2024-11-14T06:17:19,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:19,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33098 deadline: 1731565099807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:19,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:19,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33126 deadline: 1731565099807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:19,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:19,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:19,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:19,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:19,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:19,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:19,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33182 deadline: 1731565099809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:19,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:19,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:19,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:19,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:19,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:19,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33100 deadline: 1731565099817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:19,821 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:19,821 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:19,821 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:19,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742150_1326 (size=12301) 2024-11-14T06:17:19,823 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/02938b666cbd45e5975b805ac84e868f 2024-11-14T06:17:19,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/bf7418292f6a4bc2854e91d43c7c8c19 is 50, key is test_row_0/C:col10/1731565038823/Put/seqid=0 2024-11-14T06:17:19,833 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/237e924538684d0ebb83314887a34d98 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/237e924538684d0ebb83314887a34d98 2024-11-14T06:17:19,840 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5be089a788a7af479f29a70b78ed76e8/B of 5be089a788a7af479f29a70b78ed76e8 into 237e924538684d0ebb83314887a34d98(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:17:19,840 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5be089a788a7af479f29a70b78ed76e8: 2024-11-14T06:17:19,840 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., storeName=5be089a788a7af479f29a70b78ed76e8/B, priority=13, startTime=1731565039309; duration=0sec 2024-11-14T06:17:19,841 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:19,841 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5be089a788a7af479f29a70b78ed76e8:B 2024-11-14T06:17:19,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742151_1327 (size=12301) 2024-11-14T06:17:19,861 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/bf7418292f6a4bc2854e91d43c7c8c19 2024-11-14T06:17:19,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/8a402b69bece4ae9a8dc54e29751d806 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/8a402b69bece4ae9a8dc54e29751d806 2024-11-14T06:17:19,878 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/8a402b69bece4ae9a8dc54e29751d806, entries=150, sequenceid=386, filesize=12.0 K 2024-11-14T06:17:19,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/02938b666cbd45e5975b805ac84e868f as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/02938b666cbd45e5975b805ac84e868f 2024-11-14T06:17:19,884 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/02938b666cbd45e5975b805ac84e868f, entries=150, sequenceid=386, filesize=12.0 K 2024-11-14T06:17:19,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/bf7418292f6a4bc2854e91d43c7c8c19 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/bf7418292f6a4bc2854e91d43c7c8c19 2024-11-14T06:17:19,891 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/bf7418292f6a4bc2854e91d43c7c8c19, entries=150, sequenceid=386, filesize=12.0 K 2024-11-14T06:17:19,892 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 5be089a788a7af479f29a70b78ed76e8 in 557ms, sequenceid=386, compaction requested=false 2024-11-14T06:17:19,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 5be089a788a7af479f29a70b78ed76e8: 2024-11-14T06:17:19,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:19,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-14T06:17:19,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-14T06:17:19,895 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-14T06:17:19,896 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0170 sec 2024-11-14T06:17:19,899 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.0240 sec 2024-11-14T06:17:20,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-14T06:17:20,017 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:17:20,019 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T06:17:20,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-14T06:17:20,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-14T06:17:20,024 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T06:17:20,026 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T06:17:20,026 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T06:17:20,122 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5be089a788a7af479f29a70b78ed76e8 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-14T06:17:20,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on 5be089a788a7af479f29a70b78ed76e8 2024-11-14T06:17:20,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=A 2024-11-14T06:17:20,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:20,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=B 2024-11-14T06:17:20,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:20,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=C 2024-11-14T06:17:20,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:20,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-14T06:17:20,134 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/06459dd41c0d48b8835a09b4b3c78a87 is 50, key is test_row_0/A:col10/1731565040120/Put/seqid=0 2024-11-14T06:17:20,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:20,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33098 deadline: 1731565100141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:20,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:20,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:20,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:20,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:20,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33100 deadline: 1731565100147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:20,153 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:20,153 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:20,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:20,153 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:20,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33182 deadline: 1731565100149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:20,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:20,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:20,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:20,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:20,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33126 deadline: 1731565100153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:20,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:20,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:20,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:20,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742152_1328 (size=12301) 2024-11-14T06:17:20,184 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-14T06:17:20,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:20,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:20,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:20,185 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:20,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:20,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:20,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:20,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33098 deadline: 1731565100259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:20,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:20,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:20,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:20,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:20,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33100 deadline: 1731565100259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:20,264 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:20,264 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:20,264 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:20,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:20,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33182 deadline: 1731565100259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:20,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:20,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:20,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:20,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:20,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33126 deadline: 1731565100268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:20,272 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:20,272 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:20,272 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:20,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-14T06:17:20,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-14T06:17:20,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:20,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:20,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:20,341 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:20,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:20,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:20,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:20,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33098 deadline: 1731565100467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:20,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:20,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33182 deadline: 1731565100467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:20,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:20,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:20,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:20,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33100 deadline: 1731565100467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:20,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:20,471 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:20,471 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:20,471 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:20,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:20,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:20,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:20,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:20,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33126 deadline: 1731565100477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:20,484 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:20,484 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:20,484 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:20,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-14T06:17:20,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:20,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:20,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:20,494 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:20,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:20,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:20,560 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/06459dd41c0d48b8835a09b4b3c78a87 2024-11-14T06:17:20,570 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/33d53b494e594b4c8f755088042759f8 is 50, key is test_row_0/B:col10/1731565040120/Put/seqid=0 2024-11-14T06:17:20,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742153_1329 (size=12301) 2024-11-14T06:17:20,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-14T06:17:20,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-14T06:17:20,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:20,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:20,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:20,649 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:20,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:20,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:20,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:20,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33182 deadline: 1731565100777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:20,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:20,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:20,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:20,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:20,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33100 deadline: 1731565100778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:20,781 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:20,781 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:20,781 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:20,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:20,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:20,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33126 deadline: 1731565100787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:20,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33098 deadline: 1731565100787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:20,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:20,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:20,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:20,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:20,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:20,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:20,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-14T06:17:20,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:20,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:20,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:20,812 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:20,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:20,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:20,964 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-14T06:17:20,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:20,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:20,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:20,964 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:20,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:20,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:20,994 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/33d53b494e594b4c8f755088042759f8 2024-11-14T06:17:21,001 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/f2e45e34718f41cc8dc1d26a1f37df27 is 50, key is test_row_0/C:col10/1731565040120/Put/seqid=0 2024-11-14T06:17:21,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742154_1330 (size=12301) 2024-11-14T06:17:21,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-14T06:17:21,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:21,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:21,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:21,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:21,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:21,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:21,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-14T06:17:21,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-14T06:17:21,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:21,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:21,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:21,270 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:21,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:21,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:21,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:21,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33182 deadline: 1731565101287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:21,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:21,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:21,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:21,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:21,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33100 deadline: 1731565101288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:21,291 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:21,291 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:21,291 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:21,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:21,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33126 deadline: 1731565101297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:21,300 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:21,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:21,300 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:21,300 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:21,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33098 deadline: 1731565101297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:21,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:21,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:21,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:21,418 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/f2e45e34718f41cc8dc1d26a1f37df27 2024-11-14T06:17:21,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/06459dd41c0d48b8835a09b4b3c78a87 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/06459dd41c0d48b8835a09b4b3c78a87 2024-11-14T06:17:21,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-14T06:17:21,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:21,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:21,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:21,423 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:21,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:21,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:21,425 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/06459dd41c0d48b8835a09b4b3c78a87, entries=150, sequenceid=409, filesize=12.0 K 2024-11-14T06:17:21,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/33d53b494e594b4c8f755088042759f8 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/33d53b494e594b4c8f755088042759f8 2024-11-14T06:17:21,429 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/33d53b494e594b4c8f755088042759f8, entries=150, sequenceid=409, filesize=12.0 K 2024-11-14T06:17:21,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/f2e45e34718f41cc8dc1d26a1f37df27 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/f2e45e34718f41cc8dc1d26a1f37df27 2024-11-14T06:17:21,433 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/f2e45e34718f41cc8dc1d26a1f37df27, entries=150, sequenceid=409, filesize=12.0 K 2024-11-14T06:17:21,433 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 5be089a788a7af479f29a70b78ed76e8 in 1311ms, sequenceid=409, compaction requested=true 2024-11-14T06:17:21,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5be089a788a7af479f29a70b78ed76e8: 2024-11-14T06:17:21,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5be089a788a7af479f29a70b78ed76e8:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T06:17:21,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:21,434 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:17:21,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5be089a788a7af479f29a70b78ed76e8:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T06:17:21,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:21,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5be089a788a7af479f29a70b78ed76e8:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T06:17:21,434 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:17:21,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:17:21,434 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:17:21,434 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:17:21,434 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): 5be089a788a7af479f29a70b78ed76e8/B is initiating minor compaction (all files) 2024-11-14T06:17:21,434 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): 5be089a788a7af479f29a70b78ed76e8/A is initiating minor compaction (all files) 2024-11-14T06:17:21,435 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5be089a788a7af479f29a70b78ed76e8/B in TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:21,435 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/237e924538684d0ebb83314887a34d98, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/02938b666cbd45e5975b805ac84e868f, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/33d53b494e594b4c8f755088042759f8] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp, totalSize=36.9 K 2024-11-14T06:17:21,435 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5be089a788a7af479f29a70b78ed76e8/A in TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:21,435 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/f2783c99e62240a8bc21b728897811ec, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/8a402b69bece4ae9a8dc54e29751d806, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/06459dd41c0d48b8835a09b4b3c78a87] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp, totalSize=36.9 K 2024-11-14T06:17:21,435 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 237e924538684d0ebb83314887a34d98, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731565038675 2024-11-14T06:17:21,435 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2783c99e62240a8bc21b728897811ec, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731565038675 2024-11-14T06:17:21,436 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 02938b666cbd45e5975b805ac84e868f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1731565038821 2024-11-14T06:17:21,436 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a402b69bece4ae9a8dc54e29751d806, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1731565038821 2024-11-14T06:17:21,436 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 33d53b494e594b4c8f755088042759f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731565039482 2024-11-14T06:17:21,436 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06459dd41c0d48b8835a09b4b3c78a87, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731565039482 2024-11-14T06:17:21,443 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5be089a788a7af479f29a70b78ed76e8#B#compaction#285 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:17:21,443 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5be089a788a7af479f29a70b78ed76e8#A#compaction#286 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:17:21,443 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/6914b3be152d487bbc32b4b7f2f8a046 is 50, key is test_row_0/B:col10/1731565040120/Put/seqid=0 2024-11-14T06:17:21,444 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/8770146c6c29447abb34e29126ed99df is 50, key is test_row_0/A:col10/1731565040120/Put/seqid=0 2024-11-14T06:17:21,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742156_1332 (size=13323) 2024-11-14T06:17:21,453 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/8770146c6c29447abb34e29126ed99df as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/8770146c6c29447abb34e29126ed99df 2024-11-14T06:17:21,458 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5be089a788a7af479f29a70b78ed76e8/A of 5be089a788a7af479f29a70b78ed76e8 into 8770146c6c29447abb34e29126ed99df(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:17:21,458 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5be089a788a7af479f29a70b78ed76e8: 2024-11-14T06:17:21,458 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., storeName=5be089a788a7af479f29a70b78ed76e8/A, priority=13, startTime=1731565041434; duration=0sec 2024-11-14T06:17:21,458 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:17:21,458 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5be089a788a7af479f29a70b78ed76e8:A 2024-11-14T06:17:21,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742155_1331 (size=13323) 2024-11-14T06:17:21,458 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:17:21,460 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:17:21,460 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): 5be089a788a7af479f29a70b78ed76e8/C is initiating minor compaction (all files) 2024-11-14T06:17:21,460 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5be089a788a7af479f29a70b78ed76e8/C in TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:21,460 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/f15d29215aa94445890cd47ebfb53811, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/bf7418292f6a4bc2854e91d43c7c8c19, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/f2e45e34718f41cc8dc1d26a1f37df27] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp, totalSize=36.9 K 2024-11-14T06:17:21,460 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting f15d29215aa94445890cd47ebfb53811, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731565038675 2024-11-14T06:17:21,461 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf7418292f6a4bc2854e91d43c7c8c19, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1731565038821 2024-11-14T06:17:21,461 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2e45e34718f41cc8dc1d26a1f37df27, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731565039482 2024-11-14T06:17:21,464 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/6914b3be152d487bbc32b4b7f2f8a046 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/6914b3be152d487bbc32b4b7f2f8a046 2024-11-14T06:17:21,468 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5be089a788a7af479f29a70b78ed76e8#C#compaction#287 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:17:21,469 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/9b4b2400ef1145aaa9ac28e8bd90ac21 is 50, key is test_row_0/C:col10/1731565040120/Put/seqid=0 2024-11-14T06:17:21,469 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5be089a788a7af479f29a70b78ed76e8/B of 5be089a788a7af479f29a70b78ed76e8 into 6914b3be152d487bbc32b4b7f2f8a046(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:17:21,469 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5be089a788a7af479f29a70b78ed76e8: 2024-11-14T06:17:21,469 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., storeName=5be089a788a7af479f29a70b78ed76e8/B, priority=13, startTime=1731565041434; duration=0sec 2024-11-14T06:17:21,469 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:21,469 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5be089a788a7af479f29a70b78ed76e8:B 2024-11-14T06:17:21,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742157_1333 (size=13323) 2024-11-14T06:17:21,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-14T06:17:21,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:21,576 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 5be089a788a7af479f29a70b78ed76e8 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-14T06:17:21,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=A 2024-11-14T06:17:21,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:21,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=B 2024-11-14T06:17:21,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:21,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=C 2024-11-14T06:17:21,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:21,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/82f63354094f4c24a2a8789b44be674e is 50, key is test_row_0/A:col10/1731565040134/Put/seqid=0 2024-11-14T06:17:21,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742158_1334 (size=12301) 2024-11-14T06:17:21,888 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/9b4b2400ef1145aaa9ac28e8bd90ac21 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/9b4b2400ef1145aaa9ac28e8bd90ac21 2024-11-14T06:17:21,897 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5be089a788a7af479f29a70b78ed76e8/C of 5be089a788a7af479f29a70b78ed76e8 into 9b4b2400ef1145aaa9ac28e8bd90ac21(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:17:21,897 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5be089a788a7af479f29a70b78ed76e8: 2024-11-14T06:17:21,897 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., storeName=5be089a788a7af479f29a70b78ed76e8/C, priority=13, startTime=1731565041434; duration=0sec 2024-11-14T06:17:21,897 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:21,897 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5be089a788a7af479f29a70b78ed76e8:C 2024-11-14T06:17:22,005 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/82f63354094f4c24a2a8789b44be674e 2024-11-14T06:17:22,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/8dc8b202b8f4431aa6e4292cfe963986 is 50, key is test_row_0/B:col10/1731565040134/Put/seqid=0 2024-11-14T06:17:22,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742159_1335 (size=12301) 2024-11-14T06:17:22,036 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/8dc8b202b8f4431aa6e4292cfe963986 2024-11-14T06:17:22,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/870726bf50cc481d87cf38ab1285a17f is 50, key is test_row_0/C:col10/1731565040134/Put/seqid=0 2024-11-14T06:17:22,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742160_1336 (size=12301) 2024-11-14T06:17:22,069 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/870726bf50cc481d87cf38ab1285a17f 2024-11-14T06:17:22,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/82f63354094f4c24a2a8789b44be674e as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/82f63354094f4c24a2a8789b44be674e 2024-11-14T06:17:22,078 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/82f63354094f4c24a2a8789b44be674e, entries=150, sequenceid=424, filesize=12.0 K 2024-11-14T06:17:22,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/8dc8b202b8f4431aa6e4292cfe963986 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/8dc8b202b8f4431aa6e4292cfe963986 2024-11-14T06:17:22,084 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/8dc8b202b8f4431aa6e4292cfe963986, entries=150, sequenceid=424, filesize=12.0 K 2024-11-14T06:17:22,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/870726bf50cc481d87cf38ab1285a17f as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/870726bf50cc481d87cf38ab1285a17f 2024-11-14T06:17:22,089 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/870726bf50cc481d87cf38ab1285a17f, entries=150, sequenceid=424, filesize=12.0 K 2024-11-14T06:17:22,090 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=0 B/0 for 5be089a788a7af479f29a70b78ed76e8 in 514ms, sequenceid=424, compaction requested=false 2024-11-14T06:17:22,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 5be089a788a7af479f29a70b78ed76e8: 2024-11-14T06:17:22,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:22,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-14T06:17:22,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-14T06:17:22,101 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-14T06:17:22,101 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0650 sec 2024-11-14T06:17:22,105 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 2.0840 sec 2024-11-14T06:17:22,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-14T06:17:22,167 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:17:22,168 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T06:17:22,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-14T06:17:22,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-14T06:17:22,170 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T06:17:22,171 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T06:17:22,171 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T06:17:22,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-14T06:17:22,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on 5be089a788a7af479f29a70b78ed76e8 2024-11-14T06:17:22,310 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5be089a788a7af479f29a70b78ed76e8 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T06:17:22,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=A 2024-11-14T06:17:22,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:22,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=B 2024-11-14T06:17:22,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:22,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=C 2024-11-14T06:17:22,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:22,314 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/252d1aa896fb426da39819e8e25663b1 is 50, key is test_row_0/A:col10/1731565042305/Put/seqid=0 2024-11-14T06:17:22,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-14T06:17:22,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:22,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:22,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:22,326 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:22,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:22,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:22,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742161_1337 (size=14741) 2024-11-14T06:17:22,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731565042323, max timestamp=1731565042341Num uniques 859; 2024-11-14T06:17:22,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T06:17:22,344 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731565042323, max timestamp=1731565042341Num uniques 859; 2024-11-14T06:17:22,344 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T06:17:22,345 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731565042323, max timestamp=1731565042341Num uniques 859; 2024-11-14T06:17:22,345 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-14T06:17:22,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:22,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33126 deadline: 1731565102362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:22,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:22,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33100 deadline: 1731565102362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:22,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:22,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:22,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:22,368 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:22,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:22,368 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:22,368 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:22,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33098 deadline: 1731565102363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:22,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:22,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:22,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:22,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:22,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33182 deadline: 1731565102366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:22,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:22,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:22,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:22,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-14T06:17:22,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:22,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:22,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:22,480 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:22,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:22,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:22,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:22,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33126 deadline: 1731565102477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:22,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:22,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33100 deadline: 1731565102477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:22,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:22,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:22,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33098 deadline: 1731565102477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:22,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:22,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:22,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:22,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:22,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:22,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33182 deadline: 1731565102478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:22,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:22,482 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:22,482 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:22,482 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:22,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:22,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:22,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:22,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-14T06:17:22,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-14T06:17:22,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:22,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:22,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:22,633 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:22,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:22,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:22,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:22,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33126 deadline: 1731565102687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:22,692 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:22,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:22,692 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:22,692 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:22,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33098 deadline: 1731565102688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:22,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:22,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:22,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:22,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33100 deadline: 1731565102688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:22,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:22,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:22,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:22,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:22,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:22,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33182 deadline: 1731565102688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:22,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:22,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:22,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:22,736 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/252d1aa896fb426da39819e8e25663b1 2024-11-14T06:17:22,742 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/6a5d2f0a7aad4ed5853973ed72a5b0a7 is 50, key is test_row_0/B:col10/1731565042305/Put/seqid=0 2024-11-14T06:17:22,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742162_1338 (size=12301) 2024-11-14T06:17:22,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:22,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33168 deadline: 1731565102767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:22,770 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to ef93ef3a83c5,42283,1731564965945 for 'test_row_1' in 5be089a788a7af479f29a70b78ed76e8 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18295 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T06:17:22,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:22,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:22,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:22,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-14T06:17:22,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:22,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:22,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:22,786 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:22,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:22,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:22,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-14T06:17:22,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-14T06:17:22,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:22,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:22,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:22,939 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:22,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:22,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:22,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33126 deadline: 1731565102997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:22,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:22,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:22,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:22,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:22,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33098 deadline: 1731565102997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:23,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:23,000 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:23,000 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:23,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33100 deadline: 1731565102998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:23,000 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:23,000 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:23,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:23,000 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:23,000 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:23,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33182 deadline: 1731565102998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:23,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:23,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:23,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:23,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-14T06:17:23,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:23,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:23,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:23,092 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:23,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:23,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:23,146 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/6a5d2f0a7aad4ed5853973ed72a5b0a7 2024-11-14T06:17:23,151 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/9474c073a67e49cb9c111dc388d21ad7 is 50, key is test_row_0/C:col10/1731565042305/Put/seqid=0 2024-11-14T06:17:23,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742163_1339 (size=12301) 2024-11-14T06:17:23,243 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-14T06:17:23,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:23,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:23,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:23,244 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:23,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:23,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:23,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-14T06:17:23,396 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-14T06:17:23,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:23,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:23,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:23,397 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:23,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:23,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:23,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:23,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33098 deadline: 1731565103507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:23,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:23,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:23,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33126 deadline: 1731565103507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:23,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:23,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:23,511 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:23,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:23,511 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:23,511 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:23,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33182 deadline: 1731565103507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:23,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:23,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:23,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:23,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33100 deadline: 1731565103507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:23,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:23,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:23,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:23,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:17:23,549 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-14T06:17:23,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:23,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:23,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:23,550 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:23,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:23,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:23,555 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/9474c073a67e49cb9c111dc388d21ad7 2024-11-14T06:17:23,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/252d1aa896fb426da39819e8e25663b1 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/252d1aa896fb426da39819e8e25663b1 2024-11-14T06:17:23,566 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/252d1aa896fb426da39819e8e25663b1, entries=200, sequenceid=435, filesize=14.4 K 2024-11-14T06:17:23,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/6a5d2f0a7aad4ed5853973ed72a5b0a7 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/6a5d2f0a7aad4ed5853973ed72a5b0a7 2024-11-14T06:17:23,570 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/6a5d2f0a7aad4ed5853973ed72a5b0a7, entries=150, sequenceid=435, filesize=12.0 K 2024-11-14T06:17:23,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/9474c073a67e49cb9c111dc388d21ad7 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/9474c073a67e49cb9c111dc388d21ad7 2024-11-14T06:17:23,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/9474c073a67e49cb9c111dc388d21ad7, entries=150, sequenceid=435, filesize=12.0 K 2024-11-14T06:17:23,576 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=131.55 KB/134705 for 5be089a788a7af479f29a70b78ed76e8 in 1266ms, sequenceid=435, compaction requested=true 2024-11-14T06:17:23,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5be089a788a7af479f29a70b78ed76e8: 2024-11-14T06:17:23,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5be089a788a7af479f29a70b78ed76e8:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T06:17:23,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:23,576 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:17:23,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5be089a788a7af479f29a70b78ed76e8:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T06:17:23,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:23,576 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:17:23,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5be089a788a7af479f29a70b78ed76e8:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T06:17:23,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:17:23,577 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40365 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:17:23,577 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:17:23,577 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): 5be089a788a7af479f29a70b78ed76e8/A is initiating minor compaction (all files) 2024-11-14T06:17:23,577 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): 5be089a788a7af479f29a70b78ed76e8/B is initiating minor compaction (all files) 2024-11-14T06:17:23,577 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5be089a788a7af479f29a70b78ed76e8/B in TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:23,577 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5be089a788a7af479f29a70b78ed76e8/A in TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:23,577 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/6914b3be152d487bbc32b4b7f2f8a046, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/8dc8b202b8f4431aa6e4292cfe963986, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/6a5d2f0a7aad4ed5853973ed72a5b0a7] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp, totalSize=37.0 K 2024-11-14T06:17:23,577 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/8770146c6c29447abb34e29126ed99df, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/82f63354094f4c24a2a8789b44be674e, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/252d1aa896fb426da39819e8e25663b1] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp, totalSize=39.4 K 2024-11-14T06:17:23,578 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 6914b3be152d487bbc32b4b7f2f8a046, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731565039482 2024-11-14T06:17:23,578 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8770146c6c29447abb34e29126ed99df, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731565039482 2024-11-14T06:17:23,578 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 8dc8b202b8f4431aa6e4292cfe963986, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1731565040134 2024-11-14T06:17:23,578 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82f63354094f4c24a2a8789b44be674e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1731565040134 2024-11-14T06:17:23,578 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 252d1aa896fb426da39819e8e25663b1, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1731565042298 2024-11-14T06:17:23,578 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a5d2f0a7aad4ed5853973ed72a5b0a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1731565042305 2024-11-14T06:17:23,585 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5be089a788a7af479f29a70b78ed76e8#B#compaction#294 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:17:23,586 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5be089a788a7af479f29a70b78ed76e8#A#compaction#295 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:17:23,586 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/f75fa3e089e64fa6a27cb8772be62661 is 50, key is test_row_0/B:col10/1731565042305/Put/seqid=0 2024-11-14T06:17:23,586 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/da4ec7f9d16e47949523551391c48f62 is 50, key is test_row_0/A:col10/1731565042305/Put/seqid=0 2024-11-14T06:17:23,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742164_1340 (size=13425) 2024-11-14T06:17:23,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742165_1341 (size=13425) 2024-11-14T06:17:23,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-14T06:17:23,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:23,702 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 5be089a788a7af479f29a70b78ed76e8 3/3 column families, dataSize=131.55 KB heapSize=374.30 KB 2024-11-14T06:17:23,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=A 2024-11-14T06:17:23,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:17:23,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=B 2024-11-14T06:17:23,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:17:23,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=C 2024-11-14T06:17:23,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:17:23,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/6836e40f58444e8da90e8199d3eff539 is 50, key is test_row_0/A:col10/1731565042365/Put/seqid=0 2024-11-14T06:17:23,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742166_1342 (size=12301) 2024-11-14T06:17:23,726 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=471 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/6836e40f58444e8da90e8199d3eff539 2024-11-14T06:17:23,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/79eda32405144e35abe17698a7d505ce is 50, key is test_row_0/B:col10/1731565042365/Put/seqid=0 2024-11-14T06:17:23,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742167_1343 (size=12301) 2024-11-14T06:17:24,012 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/da4ec7f9d16e47949523551391c48f62 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/da4ec7f9d16e47949523551391c48f62 2024-11-14T06:17:24,016 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5be089a788a7af479f29a70b78ed76e8/A of 5be089a788a7af479f29a70b78ed76e8 into da4ec7f9d16e47949523551391c48f62(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:17:24,016 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5be089a788a7af479f29a70b78ed76e8: 2024-11-14T06:17:24,016 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., storeName=5be089a788a7af479f29a70b78ed76e8/A, priority=13, startTime=1731565043576; duration=0sec 2024-11-14T06:17:24,016 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:17:24,016 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5be089a788a7af479f29a70b78ed76e8:A 2024-11-14T06:17:24,016 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:17:24,017 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:17:24,017 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): 5be089a788a7af479f29a70b78ed76e8/C is initiating minor compaction (all files) 2024-11-14T06:17:24,017 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5be089a788a7af479f29a70b78ed76e8/C in TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:24,017 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/9b4b2400ef1145aaa9ac28e8bd90ac21, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/870726bf50cc481d87cf38ab1285a17f, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/9474c073a67e49cb9c111dc388d21ad7] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp, totalSize=37.0 K 2024-11-14T06:17:24,018 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b4b2400ef1145aaa9ac28e8bd90ac21, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731565039482 2024-11-14T06:17:24,018 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 870726bf50cc481d87cf38ab1285a17f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1731565040134 2024-11-14T06:17:24,018 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9474c073a67e49cb9c111dc388d21ad7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1731565042305 2024-11-14T06:17:24,024 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/f75fa3e089e64fa6a27cb8772be62661 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/f75fa3e089e64fa6a27cb8772be62661 2024-11-14T06:17:24,027 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5be089a788a7af479f29a70b78ed76e8#C#compaction#298 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:17:24,027 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/8556c5108147488dbb183531305de423 is 50, key is test_row_0/C:col10/1731565042305/Put/seqid=0 2024-11-14T06:17:24,029 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5be089a788a7af479f29a70b78ed76e8/B of 5be089a788a7af479f29a70b78ed76e8 into f75fa3e089e64fa6a27cb8772be62661(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:17:24,029 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5be089a788a7af479f29a70b78ed76e8: 2024-11-14T06:17:24,029 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., storeName=5be089a788a7af479f29a70b78ed76e8/B, priority=13, startTime=1731565043576; duration=0sec 2024-11-14T06:17:24,029 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:24,029 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5be089a788a7af479f29a70b78ed76e8:B 2024-11-14T06:17:24,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742168_1344 (size=13425) 2024-11-14T06:17:24,040 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/8556c5108147488dbb183531305de423 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/8556c5108147488dbb183531305de423 2024-11-14T06:17:24,045 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5be089a788a7af479f29a70b78ed76e8/C of 5be089a788a7af479f29a70b78ed76e8 into 8556c5108147488dbb183531305de423(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:17:24,045 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5be089a788a7af479f29a70b78ed76e8: 2024-11-14T06:17:24,045 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., storeName=5be089a788a7af479f29a70b78ed76e8/C, priority=13, startTime=1731565043576; duration=0sec 2024-11-14T06:17:24,045 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:24,045 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5be089a788a7af479f29a70b78ed76e8:C 2024-11-14T06:17:24,144 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=471 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/79eda32405144e35abe17698a7d505ce 2024-11-14T06:17:24,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/4fe17757c3e8423bad0b2defe6753751 is 50, key is test_row_0/C:col10/1731565042365/Put/seqid=0 2024-11-14T06:17:24,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742169_1345 (size=12301) 2024-11-14T06:17:24,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-14T06:17:24,485 INFO [Thread-1133 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1133. 2024-11-14T06:17:24,485 INFO [Thread-1135 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1135. 2024-11-14T06:17:24,485 DEBUG [Thread-1135 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T06:17:24,485 DEBUG [Thread-1133 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T06:17:24,485 DEBUG [Thread-1135 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:17:24,485 DEBUG [Thread-1133 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:17:24,485 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:17:24,485 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:17:24,485 DEBUG [Thread-1133 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:17:24,485 DEBUG [Thread-1135 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:17:24,487 INFO [Thread-1131 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1131. 2024-11-14T06:17:24,488 DEBUG [Thread-1131 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T06:17:24,488 DEBUG [Thread-1131 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:17:24,488 DEBUG [Thread-1131 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:17:24,488 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:17:24,488 INFO [Thread-1129 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1129. 2024-11-14T06:17:24,488 DEBUG [Thread-1129 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T06:17:24,488 DEBUG [Thread-1129 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:17:24,488 DEBUG [Thread-1129 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:17:24,488 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:17:24,488 INFO [Thread-1137 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1137. 2024-11-14T06:17:24,488 DEBUG [Thread-1137 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T06:17:24,488 DEBUG [Thread-1137 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:17:24,488 DEBUG [Thread-1137 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:17:24,489 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:17:24,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on 5be089a788a7af479f29a70b78ed76e8 2024-11-14T06:17:24,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. as already flushing 2024-11-14T06:17:24,518 INFO [Thread-1126 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1126. 2024-11-14T06:17:24,518 INFO [Thread-1120 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1120. 2024-11-14T06:17:24,518 INFO [Thread-1124 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1124. 2024-11-14T06:17:24,518 DEBUG [Thread-1126 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T06:17:24,518 DEBUG [Thread-1120 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T06:17:24,518 DEBUG [Thread-1124 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T06:17:24,518 DEBUG [Thread-1124 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:17:24,518 DEBUG [Thread-1120 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:17:24,518 DEBUG [Thread-1126 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:17:24,518 DEBUG [Thread-1120 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:17:24,518 DEBUG [Thread-1126 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:17:24,518 DEBUG [Thread-1124 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:17:24,518 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:17:24,519 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:17:24,519 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:17:24,529 INFO [Thread-1118 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1118. 2024-11-14T06:17:24,529 DEBUG [Thread-1118 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T06:17:24,529 DEBUG [Thread-1118 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:17:24,529 DEBUG [Thread-1118 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:17:24,529 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:17:24,555 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.09 KB at sequenceid=471 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/4fe17757c3e8423bad0b2defe6753751 2024-11-14T06:17:24,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/6836e40f58444e8da90e8199d3eff539 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/6836e40f58444e8da90e8199d3eff539 2024-11-14T06:17:24,566 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/6836e40f58444e8da90e8199d3eff539, entries=150, sequenceid=471, filesize=12.0 K 2024-11-14T06:17:24,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/79eda32405144e35abe17698a7d505ce as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/79eda32405144e35abe17698a7d505ce 2024-11-14T06:17:24,569 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/79eda32405144e35abe17698a7d505ce, entries=150, sequenceid=471, filesize=12.0 K 2024-11-14T06:17:24,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/4fe17757c3e8423bad0b2defe6753751 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/4fe17757c3e8423bad0b2defe6753751 2024-11-14T06:17:24,574 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/4fe17757c3e8423bad0b2defe6753751, entries=150, sequenceid=471, filesize=12.0 K 2024-11-14T06:17:24,575 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~131.55 KB/134705, heapSize ~374.26 KB/383240, currentSize=26.84 KB/27480 for 5be089a788a7af479f29a70b78ed76e8 in 873ms, sequenceid=471, compaction requested=false 2024-11-14T06:17:24,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 5be089a788a7af479f29a70b78ed76e8: 2024-11-14T06:17:24,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:24,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-14T06:17:24,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-14T06:17:24,578 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-14T06:17:24,578 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4050 sec 2024-11-14T06:17:24,579 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 2.4100 sec 2024-11-14T06:17:26,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-14T06:17:26,338 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:17:32,869 INFO [Thread-1122 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1122. 2024-11-14T06:17:32,869 DEBUG [Thread-1122 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T06:17:32,870 DEBUG [Thread-1122 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:17:32,870 DEBUG [Thread-1122 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:17:32,870 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:17:32,870 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T06:17:32,870 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-11-14T06:17:32,870 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 85 2024-11-14T06:17:32,870 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 3 2024-11-14T06:17:32,870 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-11-14T06:17:32,870 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 141 2024-11-14T06:17:32,870 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T06:17:32,870 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2668 2024-11-14T06:17:32,870 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2552 2024-11-14T06:17:32,870 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2577 2024-11-14T06:17:32,870 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2650 2024-11-14T06:17:32,870 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2568 2024-11-14T06:17:32,870 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T06:17:32,871 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T06:17:32,871 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T06:17:32,871 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:17:32,871 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:17:32,871 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:17:32,872 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-14T06:17:32,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T06:17:32,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-14T06:17:32,878 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731565052878"}]},"ts":"1731565052878"} 2024-11-14T06:17:32,880 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T06:17:32,881 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T06:17:32,882 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T06:17:32,884 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5be089a788a7af479f29a70b78ed76e8, UNASSIGN}] 2024-11-14T06:17:32,885 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5be089a788a7af479f29a70b78ed76e8, UNASSIGN 2024-11-14T06:17:32,885 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=5be089a788a7af479f29a70b78ed76e8, regionState=CLOSING, regionLocation=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:32,887 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5be089a788a7af479f29a70b78ed76e8, UNASSIGN because future has completed 2024-11-14T06:17:32,887 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T06:17:32,887 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945}] 2024-11-14T06:17:32,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-14T06:17:33,040 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(122): Close 5be089a788a7af479f29a70b78ed76e8 2024-11-14T06:17:33,040 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T06:17:33,041 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1722): Closing 5be089a788a7af479f29a70b78ed76e8, disabling compactions & flushes 2024-11-14T06:17:33,041 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:33,041 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:33,041 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. after waiting 0 ms 2024-11-14T06:17:33,041 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:33,042 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(2902): Flushing 5be089a788a7af479f29a70b78ed76e8 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-14T06:17:33,042 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=A 2024-11-14T06:17:33,042 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:33,043 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=B 2024-11-14T06:17:33,043 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:33,043 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5be089a788a7af479f29a70b78ed76e8, store=C 2024-11-14T06:17:33,043 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:33,049 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/d5d60e4d738d4924a080cdadc984dd07 is 50, key is test_row_0/A:col10/1731565044527/Put/seqid=0 2024-11-14T06:17:33,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742170_1346 (size=12301) 2024-11-14T06:17:33,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-14T06:17:33,455 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=482 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/d5d60e4d738d4924a080cdadc984dd07 2024-11-14T06:17:33,468 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/7059bdb7bd814bbfb232f3c490ae8726 is 50, key is test_row_0/B:col10/1731565044527/Put/seqid=0 2024-11-14T06:17:33,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742171_1347 (size=12301) 2024-11-14T06:17:33,472 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=482 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/7059bdb7bd814bbfb232f3c490ae8726 2024-11-14T06:17:33,492 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/5c41f022fdfb4d889facded7756e0b0b is 50, key is test_row_0/C:col10/1731565044527/Put/seqid=0 2024-11-14T06:17:33,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742172_1348 (size=12301) 2024-11-14T06:17:33,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-14T06:17:33,895 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=482 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/5c41f022fdfb4d889facded7756e0b0b 2024-11-14T06:17:33,899 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/A/d5d60e4d738d4924a080cdadc984dd07 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/d5d60e4d738d4924a080cdadc984dd07 2024-11-14T06:17:33,902 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/d5d60e4d738d4924a080cdadc984dd07, entries=150, sequenceid=482, filesize=12.0 K 2024-11-14T06:17:33,903 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/B/7059bdb7bd814bbfb232f3c490ae8726 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/7059bdb7bd814bbfb232f3c490ae8726 2024-11-14T06:17:33,906 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/7059bdb7bd814bbfb232f3c490ae8726, entries=150, sequenceid=482, filesize=12.0 K 2024-11-14T06:17:33,907 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/.tmp/C/5c41f022fdfb4d889facded7756e0b0b as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/5c41f022fdfb4d889facded7756e0b0b 2024-11-14T06:17:33,911 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/5c41f022fdfb4d889facded7756e0b0b, entries=150, sequenceid=482, filesize=12.0 K 2024-11-14T06:17:33,912 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 5be089a788a7af479f29a70b78ed76e8 in 870ms, sequenceid=482, compaction requested=true 2024-11-14T06:17:33,912 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/c55ef1e911e44107b5163fd380f6e409, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/ac2a2d616f044090be12f8137efa9799, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/6a4d9fb2b0e64578aa4ba7258dc40415, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/071e6ee61edd4d819ee1d58a82a06124, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/43794ab4db7d4f1c88635b8004d9d3ce, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/9e43727d93334bd592b7913b3ebffc69, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/3bc774442fce4104adfd59e48d4cab74, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/c3f9633734724a7a97f03074962f6177, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/4cfe566d261e4eac8ebe79f845120c15, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/c648b48b8ed049a5a304c1b2dad09e28, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/1112d835d51549798ed05121f5fa1d8f, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/1cf990348de44708b684ec68c81fd52c, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/7eba4ce2914b45549c5ec0a43f705cff, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/89fb7ac2abe54dccaf563a00eb1cdef7, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/dd0690d3e5df4df6b54a3d2786f31ec0, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/88733b189d104e188694c396dc2f5c29, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/3d56ddde44f64fbb9180ed136cf90a88, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/52b7dcb81810469a90ec6b7a42cabcc1, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/d357c45b80ad409d9bc1a94b03a2e775, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/5a31313d94a949e5947594f484e6c68f, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/4bf47f7894b64ffb800fd328b4d7d689, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/47e6aa723f4b4e018092caa86ef47b66, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/92d9da2b91e94d9eaf0cfbf51fbf599f, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/2cbcfa7dceae4608b8ad3e11b84324fd, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/f999f07d433d40919f25da21e11b9c2d, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/e37ffec24d4c43d8b3937724e99c1aa2, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/f6b3c386ecf94e819d733d92480ec646, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/f2783c99e62240a8bc21b728897811ec, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/8a402b69bece4ae9a8dc54e29751d806, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/8770146c6c29447abb34e29126ed99df, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/06459dd41c0d48b8835a09b4b3c78a87, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/82f63354094f4c24a2a8789b44be674e, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/252d1aa896fb426da39819e8e25663b1] to archive 2024-11-14T06:17:33,913 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T06:17:33,914 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/c55ef1e911e44107b5163fd380f6e409 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/c55ef1e911e44107b5163fd380f6e409 2024-11-14T06:17:33,915 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/ac2a2d616f044090be12f8137efa9799 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/ac2a2d616f044090be12f8137efa9799 2024-11-14T06:17:33,916 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/6a4d9fb2b0e64578aa4ba7258dc40415 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/6a4d9fb2b0e64578aa4ba7258dc40415 2024-11-14T06:17:33,917 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/071e6ee61edd4d819ee1d58a82a06124 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/071e6ee61edd4d819ee1d58a82a06124 2024-11-14T06:17:33,918 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/43794ab4db7d4f1c88635b8004d9d3ce to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/43794ab4db7d4f1c88635b8004d9d3ce 2024-11-14T06:17:33,919 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/9e43727d93334bd592b7913b3ebffc69 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/9e43727d93334bd592b7913b3ebffc69 2024-11-14T06:17:33,920 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/3bc774442fce4104adfd59e48d4cab74 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/3bc774442fce4104adfd59e48d4cab74 2024-11-14T06:17:33,921 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/c3f9633734724a7a97f03074962f6177 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/c3f9633734724a7a97f03074962f6177 2024-11-14T06:17:33,921 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/4cfe566d261e4eac8ebe79f845120c15 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/4cfe566d261e4eac8ebe79f845120c15 2024-11-14T06:17:33,922 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/c648b48b8ed049a5a304c1b2dad09e28 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/c648b48b8ed049a5a304c1b2dad09e28 2024-11-14T06:17:33,923 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/1112d835d51549798ed05121f5fa1d8f to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/1112d835d51549798ed05121f5fa1d8f 2024-11-14T06:17:33,923 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/1cf990348de44708b684ec68c81fd52c to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/1cf990348de44708b684ec68c81fd52c 2024-11-14T06:17:33,924 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/7eba4ce2914b45549c5ec0a43f705cff to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/7eba4ce2914b45549c5ec0a43f705cff 2024-11-14T06:17:33,925 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/89fb7ac2abe54dccaf563a00eb1cdef7 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/89fb7ac2abe54dccaf563a00eb1cdef7 2024-11-14T06:17:33,926 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/dd0690d3e5df4df6b54a3d2786f31ec0 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/dd0690d3e5df4df6b54a3d2786f31ec0 2024-11-14T06:17:33,926 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/88733b189d104e188694c396dc2f5c29 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/88733b189d104e188694c396dc2f5c29 2024-11-14T06:17:33,927 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/3d56ddde44f64fbb9180ed136cf90a88 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/3d56ddde44f64fbb9180ed136cf90a88 2024-11-14T06:17:33,928 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/52b7dcb81810469a90ec6b7a42cabcc1 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/52b7dcb81810469a90ec6b7a42cabcc1 2024-11-14T06:17:33,929 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/d357c45b80ad409d9bc1a94b03a2e775 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/d357c45b80ad409d9bc1a94b03a2e775 2024-11-14T06:17:33,930 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/5a31313d94a949e5947594f484e6c68f to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/5a31313d94a949e5947594f484e6c68f 2024-11-14T06:17:33,930 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/4bf47f7894b64ffb800fd328b4d7d689 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/4bf47f7894b64ffb800fd328b4d7d689 2024-11-14T06:17:33,931 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/47e6aa723f4b4e018092caa86ef47b66 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/47e6aa723f4b4e018092caa86ef47b66 2024-11-14T06:17:33,932 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/92d9da2b91e94d9eaf0cfbf51fbf599f to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/92d9da2b91e94d9eaf0cfbf51fbf599f 2024-11-14T06:17:33,933 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/2cbcfa7dceae4608b8ad3e11b84324fd to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/2cbcfa7dceae4608b8ad3e11b84324fd 2024-11-14T06:17:33,933 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/f999f07d433d40919f25da21e11b9c2d to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/f999f07d433d40919f25da21e11b9c2d 2024-11-14T06:17:33,934 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/e37ffec24d4c43d8b3937724e99c1aa2 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/e37ffec24d4c43d8b3937724e99c1aa2 2024-11-14T06:17:33,935 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/f6b3c386ecf94e819d733d92480ec646 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/f6b3c386ecf94e819d733d92480ec646 2024-11-14T06:17:33,936 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/f2783c99e62240a8bc21b728897811ec to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/f2783c99e62240a8bc21b728897811ec 2024-11-14T06:17:33,937 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/8a402b69bece4ae9a8dc54e29751d806 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/8a402b69bece4ae9a8dc54e29751d806 2024-11-14T06:17:33,938 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/8770146c6c29447abb34e29126ed99df to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/8770146c6c29447abb34e29126ed99df 2024-11-14T06:17:33,939 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/06459dd41c0d48b8835a09b4b3c78a87 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/06459dd41c0d48b8835a09b4b3c78a87 2024-11-14T06:17:33,940 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/82f63354094f4c24a2a8789b44be674e to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/82f63354094f4c24a2a8789b44be674e 2024-11-14T06:17:33,941 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/252d1aa896fb426da39819e8e25663b1 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/252d1aa896fb426da39819e8e25663b1 2024-11-14T06:17:33,943 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/8028a263b8e647a39aaf38ba9a5dafb8, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/cab1dcd54d924eb190a7f2a38f1061dc, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/31842e873f1240398dd798a62db9aac3, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/bff334d5404a4512bdfaeae853b1f642, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/4a0c0d4ef65646f4b18d8cfae6e8dfc6, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/e036f0ebe0ea4ca89ac620ba92234238, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/04602e51209749a494a271df11ef8ecb, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/6533f871b537463388e7133a5065cf3d, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/db197aa520d64d10a833d309eee51cf9, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/7c6ea4143ea8482799dcc85ca510481d, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/4916ba8c4d7b4f8599da02e34c241847, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/256b8548ce5843af803b74ae0b39bc9c, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/aa13b41b14224992bf0cc31f447f4295, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/1d4feadb9469403d8a6134285273be96, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/3e7ec646d6554540b5165a57e008e3a0, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/dc96c4c8eefb4e01ba44960721538c59, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/adbe6f8c3d69443798f3707475f993e3, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/0827ed6da7584469bf98962bf2ddb17f, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/0b7ab2edf33945bea542a47cd08a1c57, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/b283850f557f43ce977c785b638f55c8, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/66f88786085e41d1bf4908e5dbbdfd9a, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/b54c4b265c00443090d76ed29aaef75e, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/6ca4b758b36c49fe848bfe8975ec91bc, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/b1cf24a4c1d0489e96ceba77a4b0e54c, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/d0fd7aa1486d478d8f8a9eea5f97dcc3, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/246ddb2498964d3692ac0bd58dbdf92c, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/237e924538684d0ebb83314887a34d98, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/a73e56a7396041fb918e466de9b4509d, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/02938b666cbd45e5975b805ac84e868f, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/6914b3be152d487bbc32b4b7f2f8a046, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/33d53b494e594b4c8f755088042759f8, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/8dc8b202b8f4431aa6e4292cfe963986, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/6a5d2f0a7aad4ed5853973ed72a5b0a7] to archive 2024-11-14T06:17:33,944 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T06:17:33,946 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/8028a263b8e647a39aaf38ba9a5dafb8 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/8028a263b8e647a39aaf38ba9a5dafb8 2024-11-14T06:17:33,947 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/cab1dcd54d924eb190a7f2a38f1061dc to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/cab1dcd54d924eb190a7f2a38f1061dc 2024-11-14T06:17:33,948 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/31842e873f1240398dd798a62db9aac3 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/31842e873f1240398dd798a62db9aac3 2024-11-14T06:17:33,949 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/bff334d5404a4512bdfaeae853b1f642 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/bff334d5404a4512bdfaeae853b1f642 2024-11-14T06:17:33,951 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/4a0c0d4ef65646f4b18d8cfae6e8dfc6 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/4a0c0d4ef65646f4b18d8cfae6e8dfc6 2024-11-14T06:17:33,952 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/e036f0ebe0ea4ca89ac620ba92234238 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/e036f0ebe0ea4ca89ac620ba92234238 2024-11-14T06:17:33,953 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/04602e51209749a494a271df11ef8ecb to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/04602e51209749a494a271df11ef8ecb 2024-11-14T06:17:33,954 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/6533f871b537463388e7133a5065cf3d to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/6533f871b537463388e7133a5065cf3d 2024-11-14T06:17:33,954 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/db197aa520d64d10a833d309eee51cf9 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/db197aa520d64d10a833d309eee51cf9 2024-11-14T06:17:33,955 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/7c6ea4143ea8482799dcc85ca510481d to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/7c6ea4143ea8482799dcc85ca510481d 2024-11-14T06:17:33,956 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/4916ba8c4d7b4f8599da02e34c241847 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/4916ba8c4d7b4f8599da02e34c241847 2024-11-14T06:17:33,957 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/256b8548ce5843af803b74ae0b39bc9c to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/256b8548ce5843af803b74ae0b39bc9c 2024-11-14T06:17:33,957 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/aa13b41b14224992bf0cc31f447f4295 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/aa13b41b14224992bf0cc31f447f4295 2024-11-14T06:17:33,958 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/1d4feadb9469403d8a6134285273be96 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/1d4feadb9469403d8a6134285273be96 2024-11-14T06:17:33,959 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/3e7ec646d6554540b5165a57e008e3a0 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/3e7ec646d6554540b5165a57e008e3a0 2024-11-14T06:17:33,959 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/dc96c4c8eefb4e01ba44960721538c59 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/dc96c4c8eefb4e01ba44960721538c59 2024-11-14T06:17:33,960 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/adbe6f8c3d69443798f3707475f993e3 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/adbe6f8c3d69443798f3707475f993e3 2024-11-14T06:17:33,961 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/0827ed6da7584469bf98962bf2ddb17f to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/0827ed6da7584469bf98962bf2ddb17f 2024-11-14T06:17:33,961 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/0b7ab2edf33945bea542a47cd08a1c57 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/0b7ab2edf33945bea542a47cd08a1c57 2024-11-14T06:17:33,962 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/b283850f557f43ce977c785b638f55c8 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/b283850f557f43ce977c785b638f55c8 2024-11-14T06:17:33,963 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/66f88786085e41d1bf4908e5dbbdfd9a to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/66f88786085e41d1bf4908e5dbbdfd9a 2024-11-14T06:17:33,963 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/b54c4b265c00443090d76ed29aaef75e to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/b54c4b265c00443090d76ed29aaef75e 2024-11-14T06:17:33,964 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/6ca4b758b36c49fe848bfe8975ec91bc to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/6ca4b758b36c49fe848bfe8975ec91bc 2024-11-14T06:17:33,965 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/b1cf24a4c1d0489e96ceba77a4b0e54c to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/b1cf24a4c1d0489e96ceba77a4b0e54c 2024-11-14T06:17:33,965 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/d0fd7aa1486d478d8f8a9eea5f97dcc3 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/d0fd7aa1486d478d8f8a9eea5f97dcc3 2024-11-14T06:17:33,966 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/246ddb2498964d3692ac0bd58dbdf92c to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/246ddb2498964d3692ac0bd58dbdf92c 2024-11-14T06:17:33,967 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/237e924538684d0ebb83314887a34d98 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/237e924538684d0ebb83314887a34d98 2024-11-14T06:17:33,967 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/a73e56a7396041fb918e466de9b4509d to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/a73e56a7396041fb918e466de9b4509d 2024-11-14T06:17:33,968 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/02938b666cbd45e5975b805ac84e868f to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/02938b666cbd45e5975b805ac84e868f 2024-11-14T06:17:33,969 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/6914b3be152d487bbc32b4b7f2f8a046 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/6914b3be152d487bbc32b4b7f2f8a046 2024-11-14T06:17:33,970 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/33d53b494e594b4c8f755088042759f8 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/33d53b494e594b4c8f755088042759f8 2024-11-14T06:17:33,970 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/8dc8b202b8f4431aa6e4292cfe963986 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/8dc8b202b8f4431aa6e4292cfe963986 2024-11-14T06:17:33,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/6a5d2f0a7aad4ed5853973ed72a5b0a7 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/6a5d2f0a7aad4ed5853973ed72a5b0a7 2024-11-14T06:17:33,972 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/331f73d6b28f46e8a19c5b0bbc0f7c91, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/c078d0bc9bdc4f8c83a3e0ef57dbccc0, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/3ed337af46fb4a4db836ed00b65c470d, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/7290ab97d49f44b0a713d472a613ac23, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/96d322b77c6f4fa9b35173df55d73d58, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/665c3bfd8382464f832c9b0623c270ed, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/85c1415983534c988b44b310f347c205, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/2b2a7c5a04d74cffa5a8d2cc36374456, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/78a988e0a4e74766a1dc6bebfa904c48, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/5dee577ec3fc40cd800a7e2b3987c71d, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/19602ce83a5b4b6abfbccfee4eaa41f6, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/9032b2a6acd14efea98a0f07c01d566b, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/8ed7c76aeffd47eda3e5a23a56db55fa, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/31cbf3aee4b04d78960b24266b6e31f5, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/b39c10af3b7b4786a72c00ce337c31ec, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/6f40653eb7284562bd5bd28a3dab7a6b, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/6c0388b8085c45d48008b46d44d9c432, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/fa9fac7eae584be3bc715211d9db5ba4, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/9e492613c9a14aa6ae215a1461804e34, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/0fc8bf4fbbad49df970bb7a3dff049e2, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/ea0ad650abca4dd6a4b98d0cb21e3c0a, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/918da577bb91456693c8dacecbbc6847, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/ad93dfc3092645729d76bab87ba5bdef, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/ecf37bf7b61242ffbff04ae6784a242e, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/97b063eb7e7a40ac94d2e04f5b9a9049, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/6067b163357e4a97a3837eb26050bf9d, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/f15d29215aa94445890cd47ebfb53811, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/078bb9e0b5264e819e27ee8289b7f9b9, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/bf7418292f6a4bc2854e91d43c7c8c19, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/9b4b2400ef1145aaa9ac28e8bd90ac21, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/f2e45e34718f41cc8dc1d26a1f37df27, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/870726bf50cc481d87cf38ab1285a17f, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/9474c073a67e49cb9c111dc388d21ad7] to archive 2024-11-14T06:17:33,973 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T06:17:33,974 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/331f73d6b28f46e8a19c5b0bbc0f7c91 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/331f73d6b28f46e8a19c5b0bbc0f7c91 2024-11-14T06:17:33,975 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/c078d0bc9bdc4f8c83a3e0ef57dbccc0 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/c078d0bc9bdc4f8c83a3e0ef57dbccc0 2024-11-14T06:17:33,975 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/3ed337af46fb4a4db836ed00b65c470d to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/3ed337af46fb4a4db836ed00b65c470d 2024-11-14T06:17:33,976 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/7290ab97d49f44b0a713d472a613ac23 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/7290ab97d49f44b0a713d472a613ac23 2024-11-14T06:17:33,977 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/96d322b77c6f4fa9b35173df55d73d58 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/96d322b77c6f4fa9b35173df55d73d58 2024-11-14T06:17:33,978 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/665c3bfd8382464f832c9b0623c270ed to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/665c3bfd8382464f832c9b0623c270ed 2024-11-14T06:17:33,979 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/85c1415983534c988b44b310f347c205 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/85c1415983534c988b44b310f347c205 2024-11-14T06:17:33,980 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/2b2a7c5a04d74cffa5a8d2cc36374456 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/2b2a7c5a04d74cffa5a8d2cc36374456 2024-11-14T06:17:33,981 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/78a988e0a4e74766a1dc6bebfa904c48 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/78a988e0a4e74766a1dc6bebfa904c48 2024-11-14T06:17:33,981 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/5dee577ec3fc40cd800a7e2b3987c71d to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/5dee577ec3fc40cd800a7e2b3987c71d 2024-11-14T06:17:33,982 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/19602ce83a5b4b6abfbccfee4eaa41f6 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/19602ce83a5b4b6abfbccfee4eaa41f6 2024-11-14T06:17:33,982 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/9032b2a6acd14efea98a0f07c01d566b to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/9032b2a6acd14efea98a0f07c01d566b 2024-11-14T06:17:33,983 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/8ed7c76aeffd47eda3e5a23a56db55fa to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/8ed7c76aeffd47eda3e5a23a56db55fa 2024-11-14T06:17:33,984 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/31cbf3aee4b04d78960b24266b6e31f5 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/31cbf3aee4b04d78960b24266b6e31f5 2024-11-14T06:17:33,984 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/b39c10af3b7b4786a72c00ce337c31ec to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/b39c10af3b7b4786a72c00ce337c31ec 2024-11-14T06:17:33,985 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/6f40653eb7284562bd5bd28a3dab7a6b to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/6f40653eb7284562bd5bd28a3dab7a6b 2024-11-14T06:17:33,986 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/6c0388b8085c45d48008b46d44d9c432 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/6c0388b8085c45d48008b46d44d9c432 2024-11-14T06:17:33,986 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/fa9fac7eae584be3bc715211d9db5ba4 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/fa9fac7eae584be3bc715211d9db5ba4 2024-11-14T06:17:33,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/9e492613c9a14aa6ae215a1461804e34 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/9e492613c9a14aa6ae215a1461804e34 2024-11-14T06:17:33,988 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/0fc8bf4fbbad49df970bb7a3dff049e2 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/0fc8bf4fbbad49df970bb7a3dff049e2 2024-11-14T06:17:33,988 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/ea0ad650abca4dd6a4b98d0cb21e3c0a to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/ea0ad650abca4dd6a4b98d0cb21e3c0a 2024-11-14T06:17:33,989 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/918da577bb91456693c8dacecbbc6847 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/918da577bb91456693c8dacecbbc6847 2024-11-14T06:17:33,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/ad93dfc3092645729d76bab87ba5bdef to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/ad93dfc3092645729d76bab87ba5bdef 2024-11-14T06:17:33,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/ecf37bf7b61242ffbff04ae6784a242e to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/ecf37bf7b61242ffbff04ae6784a242e 2024-11-14T06:17:33,991 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/97b063eb7e7a40ac94d2e04f5b9a9049 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/97b063eb7e7a40ac94d2e04f5b9a9049 2024-11-14T06:17:33,992 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/6067b163357e4a97a3837eb26050bf9d to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/6067b163357e4a97a3837eb26050bf9d 2024-11-14T06:17:33,992 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/f15d29215aa94445890cd47ebfb53811 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/f15d29215aa94445890cd47ebfb53811 2024-11-14T06:17:33,993 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/078bb9e0b5264e819e27ee8289b7f9b9 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/078bb9e0b5264e819e27ee8289b7f9b9 2024-11-14T06:17:33,994 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/bf7418292f6a4bc2854e91d43c7c8c19 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/bf7418292f6a4bc2854e91d43c7c8c19 2024-11-14T06:17:33,994 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/9b4b2400ef1145aaa9ac28e8bd90ac21 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/9b4b2400ef1145aaa9ac28e8bd90ac21 2024-11-14T06:17:33,995 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/f2e45e34718f41cc8dc1d26a1f37df27 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/f2e45e34718f41cc8dc1d26a1f37df27 2024-11-14T06:17:33,996 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/870726bf50cc481d87cf38ab1285a17f to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/870726bf50cc481d87cf38ab1285a17f 2024-11-14T06:17:33,997 DEBUG [StoreCloser-TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/9474c073a67e49cb9c111dc388d21ad7 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/9474c073a67e49cb9c111dc388d21ad7 2024-11-14T06:17:34,001 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/recovered.edits/485.seqid, newMaxSeqId=485, maxSeqId=1 2024-11-14T06:17:34,001 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. 2024-11-14T06:17:34,001 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1676): Region close journal for 5be089a788a7af479f29a70b78ed76e8: Waiting for close lock at 1731565053041Running coprocessor pre-close hooks at 1731565053041Disabling compacts and flushes for region at 1731565053041Disabling writes for close at 1731565053041Obtaining lock to block concurrent updates at 1731565053042 (+1 ms)Preparing flush snapshotting stores in 5be089a788a7af479f29a70b78ed76e8 at 1731565053042Finished memstore snapshotting TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731565053043 (+1 ms)Flushing stores of TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8. at 1731565053044 (+1 ms)Flushing 5be089a788a7af479f29a70b78ed76e8/A: creating writer at 1731565053044Flushing 5be089a788a7af479f29a70b78ed76e8/A: appending metadata at 1731565053049 (+5 ms)Flushing 5be089a788a7af479f29a70b78ed76e8/A: closing flushed file at 1731565053049Flushing 5be089a788a7af479f29a70b78ed76e8/B: creating writer at 1731565053465 (+416 ms)Flushing 5be089a788a7af479f29a70b78ed76e8/B: appending metadata at 1731565053467 (+2 ms)Flushing 5be089a788a7af479f29a70b78ed76e8/B: closing flushed file at 1731565053467Flushing 5be089a788a7af479f29a70b78ed76e8/C: creating writer at 1731565053475 (+8 ms)Flushing 5be089a788a7af479f29a70b78ed76e8/C: appending metadata at 1731565053492 (+17 ms)Flushing 5be089a788a7af479f29a70b78ed76e8/C: closing flushed file at 1731565053492Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4c2ef6f6: reopening flushed file at 1731565053898 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4dfe2b56: reopening flushed file at 1731565053902 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3182a10b: reopening flushed file at 1731565053906 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 5be089a788a7af479f29a70b78ed76e8 in 870ms, sequenceid=482, compaction requested=true at 1731565053912 (+6 ms)Writing region close event to WAL at 1731565053998 (+86 ms)Running coprocessor post-close hooks at 1731565054001 (+3 ms)Closed at 1731565054001 2024-11-14T06:17:34,003 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(157): Closed 5be089a788a7af479f29a70b78ed76e8 2024-11-14T06:17:34,003 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=5be089a788a7af479f29a70b78ed76e8, regionState=CLOSED 2024-11-14T06:17:34,004 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 because future has completed 2024-11-14T06:17:34,006 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-14T06:17:34,007 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseRegionProcedure 5be089a788a7af479f29a70b78ed76e8, server=ef93ef3a83c5,42283,1731564965945 in 1.1180 sec 2024-11-14T06:17:34,008 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81 2024-11-14T06:17:34,008 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5be089a788a7af479f29a70b78ed76e8, UNASSIGN in 1.1230 sec 2024-11-14T06:17:34,009 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-14T06:17:34,009 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.1260 sec 2024-11-14T06:17:34,010 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731565054010"}]},"ts":"1731565054010"} 2024-11-14T06:17:34,012 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T06:17:34,012 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T06:17:34,014 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.1400 sec 2024-11-14T06:17:34,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-14T06:17:34,018 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:17:34,018 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-14T06:17:34,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T06:17:34,019 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=84, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T06:17:34,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-14T06:17:34,020 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=84, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T06:17:34,023 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8 2024-11-14T06:17:34,024 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A, FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B, FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C, FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/recovered.edits] 2024-11-14T06:17:34,027 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/6836e40f58444e8da90e8199d3eff539 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/6836e40f58444e8da90e8199d3eff539 2024-11-14T06:17:34,028 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/d5d60e4d738d4924a080cdadc984dd07 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/d5d60e4d738d4924a080cdadc984dd07 2024-11-14T06:17:34,028 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/da4ec7f9d16e47949523551391c48f62 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/A/da4ec7f9d16e47949523551391c48f62 2024-11-14T06:17:34,030 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/7059bdb7bd814bbfb232f3c490ae8726 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/7059bdb7bd814bbfb232f3c490ae8726 2024-11-14T06:17:34,031 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/79eda32405144e35abe17698a7d505ce to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/79eda32405144e35abe17698a7d505ce 2024-11-14T06:17:34,032 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/f75fa3e089e64fa6a27cb8772be62661 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/B/f75fa3e089e64fa6a27cb8772be62661 2024-11-14T06:17:34,034 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/4fe17757c3e8423bad0b2defe6753751 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/4fe17757c3e8423bad0b2defe6753751 2024-11-14T06:17:34,035 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/5c41f022fdfb4d889facded7756e0b0b to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/5c41f022fdfb4d889facded7756e0b0b 2024-11-14T06:17:34,036 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/8556c5108147488dbb183531305de423 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/C/8556c5108147488dbb183531305de423 2024-11-14T06:17:34,038 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/recovered.edits/485.seqid to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8/recovered.edits/485.seqid 2024-11-14T06:17:34,038 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/5be089a788a7af479f29a70b78ed76e8 2024-11-14T06:17:34,038 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T06:17:34,040 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=84, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T06:17:34,042 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T06:17:34,044 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T06:17:34,045 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=84, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T06:17:34,045 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T06:17:34,045 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731565054045"}]},"ts":"9223372036854775807"} 2024-11-14T06:17:34,047 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T06:17:34,047 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 5be089a788a7af479f29a70b78ed76e8, NAME => 'TestAcidGuarantees,,1731565023202.5be089a788a7af479f29a70b78ed76e8.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T06:17:34,047 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T06:17:34,047 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731565054047"}]},"ts":"9223372036854775807"} 2024-11-14T06:17:34,048 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T06:17:34,049 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=84, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T06:17:34,050 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 31 msec 2024-11-14T06:17:34,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-14T06:17:34,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T06:17:34,130 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:17:34,136 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248 (was 255), OpenFileDescriptor=451 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=854 (was 884), ProcessCount=11 (was 11), AvailableMemoryMB=243 (was 544) 2024-11-14T06:17:34,145 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=248, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=854, ProcessCount=11, AvailableMemoryMB=242 2024-11-14T06:17:34,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T06:17:34,146 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T06:17:34,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=85, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T06:17:34,149 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T06:17:34,149 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:34,149 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 85 2024-11-14T06:17:34,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-14T06:17:34,150 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T06:17:34,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742173_1349 (size=1061) 2024-11-14T06:17:34,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-14T06:17:34,283 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T06:17:34,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-14T06:17:34,560 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02 2024-11-14T06:17:34,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742174_1350 (size=53) 2024-11-14T06:17:34,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-14T06:17:34,967 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T06:17:34,968 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 8b0928b43a2d9ea8c8bf8f2250ec54df, disabling compactions & flushes 2024-11-14T06:17:34,968 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:34,968 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:34,968 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. after waiting 0 ms 2024-11-14T06:17:34,968 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:34,968 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:34,968 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8b0928b43a2d9ea8c8bf8f2250ec54df: Waiting for close lock at 1731565054968Disabling compacts and flushes for region at 1731565054968Disabling writes for close at 1731565054968Writing region close event to WAL at 1731565054968Closed at 1731565054968 2024-11-14T06:17:34,971 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T06:17:34,972 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731565054971"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731565054971"}]},"ts":"1731565054971"} 2024-11-14T06:17:34,976 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T06:17:34,977 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T06:17:34,977 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731565054977"}]},"ts":"1731565054977"} 2024-11-14T06:17:34,978 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T06:17:34,979 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b0928b43a2d9ea8c8bf8f2250ec54df, ASSIGN}] 2024-11-14T06:17:34,980 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b0928b43a2d9ea8c8bf8f2250ec54df, ASSIGN 2024-11-14T06:17:34,981 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b0928b43a2d9ea8c8bf8f2250ec54df, ASSIGN; state=OFFLINE, location=ef93ef3a83c5,42283,1731564965945; forceNewPlan=false, retain=false 2024-11-14T06:17:35,131 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=8b0928b43a2d9ea8c8bf8f2250ec54df, regionState=OPENING, regionLocation=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:35,133 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b0928b43a2d9ea8c8bf8f2250ec54df, ASSIGN because future has completed 2024-11-14T06:17:35,133 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945}] 2024-11-14T06:17:35,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-14T06:17:35,290 INFO [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:35,290 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7752): Opening region: {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} 2024-11-14T06:17:35,291 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:35,291 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T06:17:35,291 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7794): checking encryption for 8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:35,291 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7797): checking classloading for 8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:35,294 INFO [StoreOpener-8b0928b43a2d9ea8c8bf8f2250ec54df-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:35,296 INFO [StoreOpener-8b0928b43a2d9ea8c8bf8f2250ec54df-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T06:17:35,296 INFO [StoreOpener-8b0928b43a2d9ea8c8bf8f2250ec54df-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8b0928b43a2d9ea8c8bf8f2250ec54df columnFamilyName A 2024-11-14T06:17:35,297 DEBUG [StoreOpener-8b0928b43a2d9ea8c8bf8f2250ec54df-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:35,297 INFO [StoreOpener-8b0928b43a2d9ea8c8bf8f2250ec54df-1 {}] regionserver.HStore(327): Store=8b0928b43a2d9ea8c8bf8f2250ec54df/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T06:17:35,297 INFO [StoreOpener-8b0928b43a2d9ea8c8bf8f2250ec54df-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:35,298 INFO [StoreOpener-8b0928b43a2d9ea8c8bf8f2250ec54df-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T06:17:35,299 INFO [StoreOpener-8b0928b43a2d9ea8c8bf8f2250ec54df-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8b0928b43a2d9ea8c8bf8f2250ec54df columnFamilyName B 2024-11-14T06:17:35,299 DEBUG [StoreOpener-8b0928b43a2d9ea8c8bf8f2250ec54df-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:35,299 INFO [StoreOpener-8b0928b43a2d9ea8c8bf8f2250ec54df-1 {}] regionserver.HStore(327): Store=8b0928b43a2d9ea8c8bf8f2250ec54df/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T06:17:35,299 INFO [StoreOpener-8b0928b43a2d9ea8c8bf8f2250ec54df-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:35,300 INFO [StoreOpener-8b0928b43a2d9ea8c8bf8f2250ec54df-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T06:17:35,300 INFO [StoreOpener-8b0928b43a2d9ea8c8bf8f2250ec54df-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8b0928b43a2d9ea8c8bf8f2250ec54df columnFamilyName C 2024-11-14T06:17:35,300 DEBUG [StoreOpener-8b0928b43a2d9ea8c8bf8f2250ec54df-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:35,300 INFO [StoreOpener-8b0928b43a2d9ea8c8bf8f2250ec54df-1 {}] regionserver.HStore(327): Store=8b0928b43a2d9ea8c8bf8f2250ec54df/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T06:17:35,301 INFO [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:35,301 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1038): replaying wal for 8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:35,301 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:35,301 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:35,303 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1048): stopping wal replay for 8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:35,303 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1060): Cleaning up temporary data for 8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:35,303 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T06:17:35,304 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1093): writing seq id for 8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:35,306 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T06:17:35,306 INFO [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1114): Opened 8b0928b43a2d9ea8c8bf8f2250ec54df; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61692429, jitterRate=-0.08071117103099823}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T06:17:35,306 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:35,307 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1006): Region open journal for 8b0928b43a2d9ea8c8bf8f2250ec54df: Running coprocessor pre-open hook at 1731565055291Writing region info on filesystem at 1731565055291Initializing all the Stores at 1731565055293 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731565055293Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731565055293Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731565055293Cleaning up temporary data from old regions at 1731565055303 (+10 ms)Running coprocessor post-open hooks at 1731565055306 (+3 ms)Region opened successfully at 1731565055307 (+1 ms) 2024-11-14T06:17:35,307 INFO [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., pid=87, masterSystemTime=1731565055285 2024-11-14T06:17:35,309 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:35,309 INFO [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:35,310 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=8b0928b43a2d9ea8c8bf8f2250ec54df, regionState=OPEN, openSeqNum=2, regionLocation=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:35,311 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 because future has completed 2024-11-14T06:17:35,312 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36045 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=ef93ef3a83c5,42283,1731564965945, table=TestAcidGuarantees, region=8b0928b43a2d9ea8c8bf8f2250ec54df. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-14T06:17:35,315 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-14T06:17:35,315 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; OpenRegionProcedure 8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 in 180 msec 2024-11-14T06:17:35,316 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-11-14T06:17:35,316 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b0928b43a2d9ea8c8bf8f2250ec54df, ASSIGN in 336 msec 2024-11-14T06:17:35,317 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T06:17:35,317 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731565055317"}]},"ts":"1731565055317"} 2024-11-14T06:17:35,319 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T06:17:35,320 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T06:17:35,321 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1740 sec 2024-11-14T06:17:36,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-14T06:17:36,298 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:17:36,299 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77b96aec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:36,299 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:17:36,299 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:17:36,300 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:17:36,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:17:36,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:17:36,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77d8bb45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:36,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:17:36,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:17:36,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:17:36,301 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45088, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:17:36,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68b21c50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:36,302 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:17:36,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:17:36,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:17:36,303 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57330, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:17:36,304 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T06:17:36,305 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is ef93ef3a83c5,36045,1731564965170 2024-11-14T06:17:36,305 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@65aae795 2024-11-14T06:17:36,305 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T06:17:36,305 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45090, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T06:17:36,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T06:17:36,307 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T06:17:36,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-14T06:17:36,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742175_1351 (size=1097) 2024-11-14T06:17:36,724 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-14T06:17:36,724 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-14T06:17:36,730 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T06:17:36,733 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b0928b43a2d9ea8c8bf8f2250ec54df, REOPEN/MOVE}] 2024-11-14T06:17:36,733 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b0928b43a2d9ea8c8bf8f2250ec54df, REOPEN/MOVE 2024-11-14T06:17:36,734 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=8b0928b43a2d9ea8c8bf8f2250ec54df, regionState=CLOSING, regionLocation=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:36,735 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b0928b43a2d9ea8c8bf8f2250ec54df, REOPEN/MOVE because future has completed 2024-11-14T06:17:36,736 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T06:17:36,736 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945}] 2024-11-14T06:17:36,889 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(122): Close 8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:36,889 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T06:17:36,889 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1722): Closing 8b0928b43a2d9ea8c8bf8f2250ec54df, disabling compactions & flushes 2024-11-14T06:17:36,889 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:36,889 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:36,889 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. after waiting 0 ms 2024-11-14T06:17:36,889 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:36,897 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T06:17:36,897 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:36,897 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1676): Region close journal for 8b0928b43a2d9ea8c8bf8f2250ec54df: Waiting for close lock at 1731565056889Running coprocessor pre-close hooks at 1731565056889Disabling compacts and flushes for region at 1731565056889Disabling writes for close at 1731565056889Writing region close event to WAL at 1731565056890 (+1 ms)Running coprocessor post-close hooks at 1731565056897 (+7 ms)Closed at 1731565056897 2024-11-14T06:17:36,898 WARN [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionServer(3299): Not adding moved region record: 8b0928b43a2d9ea8c8bf8f2250ec54df to self. 2024-11-14T06:17:36,900 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(157): Closed 8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:36,901 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=8b0928b43a2d9ea8c8bf8f2250ec54df, regionState=CLOSED 2024-11-14T06:17:36,902 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 because future has completed 2024-11-14T06:17:36,906 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-14T06:17:36,906 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; CloseRegionProcedure 8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 in 168 msec 2024-11-14T06:17:36,907 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b0928b43a2d9ea8c8bf8f2250ec54df, REOPEN/MOVE; state=CLOSED, location=ef93ef3a83c5,42283,1731564965945; forceNewPlan=false, retain=true 2024-11-14T06:17:37,058 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=8b0928b43a2d9ea8c8bf8f2250ec54df, regionState=OPENING, regionLocation=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:37,061 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b0928b43a2d9ea8c8bf8f2250ec54df, REOPEN/MOVE because future has completed 2024-11-14T06:17:37,062 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945}] 2024-11-14T06:17:37,219 INFO [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:37,220 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7752): Opening region: {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} 2024-11-14T06:17:37,220 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:37,220 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T06:17:37,220 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7794): checking encryption for 8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:37,220 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7797): checking classloading for 8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:37,223 INFO [StoreOpener-8b0928b43a2d9ea8c8bf8f2250ec54df-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:37,225 INFO [StoreOpener-8b0928b43a2d9ea8c8bf8f2250ec54df-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T06:17:37,225 INFO [StoreOpener-8b0928b43a2d9ea8c8bf8f2250ec54df-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8b0928b43a2d9ea8c8bf8f2250ec54df columnFamilyName A 2024-11-14T06:17:37,226 DEBUG [StoreOpener-8b0928b43a2d9ea8c8bf8f2250ec54df-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:37,226 INFO [StoreOpener-8b0928b43a2d9ea8c8bf8f2250ec54df-1 {}] regionserver.HStore(327): Store=8b0928b43a2d9ea8c8bf8f2250ec54df/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T06:17:37,227 INFO [StoreOpener-8b0928b43a2d9ea8c8bf8f2250ec54df-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:37,227 INFO [StoreOpener-8b0928b43a2d9ea8c8bf8f2250ec54df-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T06:17:37,227 INFO [StoreOpener-8b0928b43a2d9ea8c8bf8f2250ec54df-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8b0928b43a2d9ea8c8bf8f2250ec54df columnFamilyName B 2024-11-14T06:17:37,228 DEBUG [StoreOpener-8b0928b43a2d9ea8c8bf8f2250ec54df-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:37,228 INFO [StoreOpener-8b0928b43a2d9ea8c8bf8f2250ec54df-1 {}] regionserver.HStore(327): Store=8b0928b43a2d9ea8c8bf8f2250ec54df/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T06:17:37,228 INFO [StoreOpener-8b0928b43a2d9ea8c8bf8f2250ec54df-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:37,229 INFO [StoreOpener-8b0928b43a2d9ea8c8bf8f2250ec54df-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T06:17:37,229 INFO [StoreOpener-8b0928b43a2d9ea8c8bf8f2250ec54df-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8b0928b43a2d9ea8c8bf8f2250ec54df columnFamilyName C 2024-11-14T06:17:37,229 DEBUG [StoreOpener-8b0928b43a2d9ea8c8bf8f2250ec54df-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:37,229 INFO [StoreOpener-8b0928b43a2d9ea8c8bf8f2250ec54df-1 {}] regionserver.HStore(327): Store=8b0928b43a2d9ea8c8bf8f2250ec54df/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T06:17:37,230 INFO [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:37,230 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1038): replaying wal for 8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:37,230 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:37,231 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:37,232 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1048): stopping wal replay for 8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:37,232 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1060): Cleaning up temporary data for 8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:37,232 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T06:17:37,233 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1093): writing seq id for 8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:37,234 INFO [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1114): Opened 8b0928b43a2d9ea8c8bf8f2250ec54df; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59363776, jitterRate=-0.11541080474853516}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T06:17:37,234 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:37,235 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1006): Region open journal for 8b0928b43a2d9ea8c8bf8f2250ec54df: Running coprocessor pre-open hook at 1731565057220Writing region info on filesystem at 1731565057220Initializing all the Stores at 1731565057221 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731565057221Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731565057223 (+2 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731565057223Cleaning up temporary data from old regions at 1731565057232 (+9 ms)Running coprocessor post-open hooks at 1731565057234 (+2 ms)Region opened successfully at 1731565057235 (+1 ms) 2024-11-14T06:17:37,236 INFO [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., pid=92, masterSystemTime=1731565057217 2024-11-14T06:17:37,238 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:37,238 INFO [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:37,238 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=8b0928b43a2d9ea8c8bf8f2250ec54df, regionState=OPEN, openSeqNum=5, regionLocation=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:37,240 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 because future has completed 2024-11-14T06:17:37,241 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=90 2024-11-14T06:17:37,242 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=90, state=SUCCESS, hasLock=false; OpenRegionProcedure 8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 in 178 msec 2024-11-14T06:17:37,243 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-14T06:17:37,244 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b0928b43a2d9ea8c8bf8f2250ec54df, REOPEN/MOVE in 509 msec 2024-11-14T06:17:37,246 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-14T06:17:37,247 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 514 msec 2024-11-14T06:17:37,249 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 940 msec 2024-11-14T06:17:37,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-14T06:17:37,250 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:17:37,251 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f32b497, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:37,251 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:17:37,251 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:17:37,252 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:17:37,252 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:17:37,252 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:17:37,252 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20d6c0fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:37,252 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:17:37,252 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:17:37,252 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:17:37,253 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45110, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:17:37,254 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5789e770, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:37,254 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@695c024f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:37,254 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:17:37,254 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:17:37,255 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:17:37,255 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:17:37,255 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:17:37,255 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46350d13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:37,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:17:37,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:17:37,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:17:37,256 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45124, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:17:37,257 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1681e968, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:37,257 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@549586f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:37,258 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:17:37,258 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:17:37,258 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:17:37,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:17:37,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:17:37,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68ed2801, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:37,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:17:37,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:17:37,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:17:37,259 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45144, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:17:37,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31022dd2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:37,260 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cba9b20, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:37,260 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:17:37,261 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:17:37,261 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:17:37,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:17:37,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:17:37,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7de22f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:37,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:17:37,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:17:37,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:17:37,263 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45158, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:17:37,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a2f26c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:37,263 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bf917c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:37,263 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:17:37,264 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:17:37,264 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:17:37,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:17:37,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:17:37,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b892caa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:37,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:17:37,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:17:37,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:17:37,266 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45162, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:17:37,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ed50630, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:37,267 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74b29348, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:37,267 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:17:37,267 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:17:37,268 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:17:37,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:17:37,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:17:37,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e7e0471, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:37,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:17:37,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:17:37,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:17:37,269 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45184, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:17:37,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19e8b91e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:37,270 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d3b32d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:37,270 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:17:37,270 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:17:37,271 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:17:37,271 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:17:37,271 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:17:37,272 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65bd4ddb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:37,272 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:17:37,272 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:17:37,272 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:17:37,272 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45196, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:17:37,273 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c51d5ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:37,273 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b79742b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:37,273 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:17:37,274 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:17:37,274 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:17:37,274 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:17:37,274 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:17:37,274 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5062b824, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:37,274 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:17:37,275 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:17:37,275 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:17:37,275 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45206, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:17:37,276 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60094d8f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:37,276 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d85bef8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:37,276 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:17:37,276 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:17:37,277 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:17:37,277 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:17:37,277 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:17:37,278 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49f3a211, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:37,278 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:17:37,278 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:17:37,278 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:17:37,279 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45230, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:17:37,279 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b920bb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:37,280 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58271f99, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:37,280 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:17:37,280 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:17:37,281 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:17:37,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:17:37,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:17:37,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@401657f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:37,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:17:37,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:17:37,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:17:37,283 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45254, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:17:37,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59873562, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:17:37,284 DEBUG [Thread-1590 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:17:37,284 DEBUG [Thread-1592 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:17:37,284 DEBUG [Thread-1596 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:17:37,284 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T06:17:37,285 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:17:37,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:17:37,285 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:17:37,285 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:17:37,285 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:17:37,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:17:37,285 DEBUG [Thread-1601 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:17:37,285 DEBUG [Thread-1594 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:17:37,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-11-14T06:17:37,286 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57348, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:17:37,286 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T06:17:37,286 DEBUG [Thread-1607 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:17:37,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:17:37,287 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57358, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:17:37,287 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57346, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:17:37,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:17:37,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-14T06:17:37,288 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57374, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:17:37,288 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:17:37,288 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:17:37,288 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T06:17:37,288 DEBUG [Thread-1594 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:17:37,288 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T06:17:37,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5] 2024-11-14T06:17:37,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5] 2024-11-14T06:17:37,290 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57376, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:17:37,290 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5] 2024-11-14T06:17:37,291 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5] 2024-11-14T06:17:37,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5] 2024-11-14T06:17:37,293 DEBUG [Thread-1594 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:17:37,294 DEBUG [Thread-1609 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:17:37,294 DEBUG [Thread-1598 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:17:37,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:17:37,294 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57382, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:17:37,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:17:37,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:17:37,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:17:37,296 DEBUG [Thread-1605 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:17:37,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5] 2024-11-14T06:17:37,298 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57388, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:17:37,298 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:17:37,298 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:17:37,299 DEBUG [Thread-1603 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:17:37,299 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:17:37,299 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57402, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:17:37,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5] 2024-11-14T06:17:37,300 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:17:37,300 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57418, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:17:37,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5] 2024-11-14T06:17:37,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5] 2024-11-14T06:17:37,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on 8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:37,303 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8b0928b43a2d9ea8c8bf8f2250ec54df 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-14T06:17:37,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b0928b43a2d9ea8c8bf8f2250ec54df, store=A 2024-11-14T06:17:37,303 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57432, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:17:37,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:37,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b0928b43a2d9ea8c8bf8f2250ec54df, store=B 2024-11-14T06:17:37,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:37,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b0928b43a2d9ea8c8bf8f2250ec54df, store=C 2024-11-14T06:17:37,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:37,305 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5] 2024-11-14T06:17:37,337 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411148af17cd9177a416eac9fe70bbffeb4f1_8b0928b43a2d9ea8c8bf8f2250ec54df is 50, key is test_row_0/A:col10/1731565057299/Put/seqid=0 2024-11-14T06:17:37,341 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731565057312, max timestamp=1731565057338Num uniques 859; 2024-11-14T06:17:37,341 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T06:17:37,342 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731565057312, max timestamp=1731565057338Num uniques 859; 2024-11-14T06:17:37,342 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731565057312, max timestamp=1731565057338Num uniques 859; 2024-11-14T06:17:37,342 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T06:17:37,342 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T06:17:37,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742176_1352 (size=9714) 2024-11-14T06:17:37,350 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:37,354 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411148af17cd9177a416eac9fe70bbffeb4f1_8b0928b43a2d9ea8c8bf8f2250ec54df to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411148af17cd9177a416eac9fe70bbffeb4f1_8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:37,355 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/2aa41640c78144a38b1dba9f2f406934, store: [table=TestAcidGuarantees family=A region=8b0928b43a2d9ea8c8bf8f2250ec54df] 2024-11-14T06:17:37,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/2aa41640c78144a38b1dba9f2f406934 is 175, key is test_row_0/A:col10/1731565057299/Put/seqid=0 2024-11-14T06:17:37,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742177_1353 (size=22361) 2024-11-14T06:17:37,363 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/2aa41640c78144a38b1dba9f2f406934 2024-11-14T06:17:37,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:37,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57358 deadline: 1731565117370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:37,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:37,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:37,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:37,389 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/35d1c9ef29eb43df96016ebc9e8308a4 is 50, key is test_row_0/B:col10/1731565057299/Put/seqid=0 2024-11-14T06:17:37,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:37,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57346 deadline: 1731565117374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:37,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:37,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1731565117374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:37,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:37,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:37,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:37,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:37,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57382 deadline: 1731565117376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:37,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:37,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:37,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:37,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:37,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:37,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57348 deadline: 1731565117376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:37,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:37,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:37,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:37,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:37,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:37,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-14T06:17:37,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742178_1354 (size=9657) 2024-11-14T06:17:37,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-14T06:17:37,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:37,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. as already flushing 2024-11-14T06:17:37,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:37,443 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:37,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:37,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:37,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:37,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57358 deadline: 1731565117487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:37,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:37,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:37,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:37,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:37,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57346 deadline: 1731565117497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:37,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:37,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1731565117497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:37,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:37,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:37,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:37,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:37,501 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:37,501 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:37,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57382 deadline: 1731565117497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:37,501 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:37,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:37,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:37,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:37,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:37,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57348 deadline: 1731565117497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:37,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:37,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:37,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:37,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-14T06:17:37,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:37,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. as already flushing 2024-11-14T06:17:37,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:37,595 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:37,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:37,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:37,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-14T06:17:37,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:37,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57358 deadline: 1731565117697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:37,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:37,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:37,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:37,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:37,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57346 deadline: 1731565117707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:37,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:37,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:37,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1731565117707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:37,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:37,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:37,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:37,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:37,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:37,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57382 deadline: 1731565117707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:37,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:37,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:37,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:37,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:37,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:37,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57348 deadline: 1731565117707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:37,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:37,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:37,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:37,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-14T06:17:37,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:37,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. as already flushing 2024-11-14T06:17:37,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:37,747 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:37,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:37,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:37,800 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/35d1c9ef29eb43df96016ebc9e8308a4 2024-11-14T06:17:37,825 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/4d370e25165a4225b2e2b770dc5cba02 is 50, key is test_row_0/C:col10/1731565057299/Put/seqid=0 2024-11-14T06:17:37,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742179_1355 (size=9657) 2024-11-14T06:17:37,900 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-14T06:17:37,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:37,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. as already flushing 2024-11-14T06:17:37,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:37,900 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:37,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:37,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:37,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-14T06:17:38,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:38,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57358 deadline: 1731565118007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:38,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:38,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:38,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:38,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:38,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57346 deadline: 1731565118017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:38,022 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:38,022 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:38,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:38,022 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:38,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1731565118017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:38,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:38,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:38,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:38,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:38,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57382 deadline: 1731565118017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:38,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:38,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:38,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:38,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:38,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57348 deadline: 1731565118017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:38,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:38,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:38,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:38,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-14T06:17:38,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:38,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. as already flushing 2024-11-14T06:17:38,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:38,053 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:38,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:38,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:38,208 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-14T06:17:38,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:38,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. as already flushing 2024-11-14T06:17:38,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:38,209 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:38,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:38,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:38,229 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/4d370e25165a4225b2e2b770dc5cba02 2024-11-14T06:17:38,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/2aa41640c78144a38b1dba9f2f406934 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/2aa41640c78144a38b1dba9f2f406934 2024-11-14T06:17:38,240 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/2aa41640c78144a38b1dba9f2f406934, entries=100, sequenceid=15, filesize=21.8 K 2024-11-14T06:17:38,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/35d1c9ef29eb43df96016ebc9e8308a4 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/35d1c9ef29eb43df96016ebc9e8308a4 2024-11-14T06:17:38,252 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/35d1c9ef29eb43df96016ebc9e8308a4, entries=100, sequenceid=15, filesize=9.4 K 2024-11-14T06:17:38,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/4d370e25165a4225b2e2b770dc5cba02 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/4d370e25165a4225b2e2b770dc5cba02 2024-11-14T06:17:38,262 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/4d370e25165a4225b2e2b770dc5cba02, entries=100, sequenceid=15, filesize=9.4 K 2024-11-14T06:17:38,263 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 8b0928b43a2d9ea8c8bf8f2250ec54df in 960ms, sequenceid=15, compaction requested=false 2024-11-14T06:17:38,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8b0928b43a2d9ea8c8bf8f2250ec54df: 2024-11-14T06:17:38,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-14T06:17:38,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:38,364 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing 8b0928b43a2d9ea8c8bf8f2250ec54df 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-14T06:17:38,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b0928b43a2d9ea8c8bf8f2250ec54df, store=A 2024-11-14T06:17:38,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:17:38,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b0928b43a2d9ea8c8bf8f2250ec54df, store=B 2024-11-14T06:17:38,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:17:38,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b0928b43a2d9ea8c8bf8f2250ec54df, store=C 2024-11-14T06:17:38,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:17:38,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114087cf6bb8ac84f9ab54a4815ab59e629_8b0928b43a2d9ea8c8bf8f2250ec54df is 50, key is test_row_0/A:col10/1731565057371/Put/seqid=0 2024-11-14T06:17:38,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742180_1356 (size=12154) 2024-11-14T06:17:38,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:38,423 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114087cf6bb8ac84f9ab54a4815ab59e629_8b0928b43a2d9ea8c8bf8f2250ec54df to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114087cf6bb8ac84f9ab54a4815ab59e629_8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:38,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/2273c8cdaabf49118107593c6def0c72, store: [table=TestAcidGuarantees family=A region=8b0928b43a2d9ea8c8bf8f2250ec54df] 2024-11-14T06:17:38,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/2273c8cdaabf49118107593c6def0c72 is 175, key is test_row_0/A:col10/1731565057371/Put/seqid=0 2024-11-14T06:17:38,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-14T06:17:38,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742181_1357 (size=30955) 2024-11-14T06:17:38,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on 8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:38,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. as already flushing 2024-11-14T06:17:38,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:38,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57382 deadline: 1731565118547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:38,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:38,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:38,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57358 deadline: 1731565118547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:38,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:38,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:38,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:38,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:38,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:38,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:38,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1731565118549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:38,559 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:38,559 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:38,559 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:38,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:38,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:38,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57348 deadline: 1731565118550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:38,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57346 deadline: 1731565118552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:38,564 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:38,564 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:38,564 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:38,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:38,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:38,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:38,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:38,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57382 deadline: 1731565118667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:38,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:38,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57358 deadline: 1731565118668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:38,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:38,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:38,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:38,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:38,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:38,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57346 deadline: 1731565118668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:38,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:38,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:38,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:38,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:38,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:38,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1731565118668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:38,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:38,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:38,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:38,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:38,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:38,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57348 deadline: 1731565118677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:38,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:38,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:38,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:38,857 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/2273c8cdaabf49118107593c6def0c72 2024-11-14T06:17:38,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/f8bd7fab0aca4689a95aebea33bfb425 is 50, key is test_row_0/B:col10/1731565057371/Put/seqid=0 2024-11-14T06:17:38,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742182_1358 (size=12001) 2024-11-14T06:17:38,871 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/f8bd7fab0aca4689a95aebea33bfb425 2024-11-14T06:17:38,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/35915e179cf842ce97d461a72b584c2a is 50, key is test_row_0/C:col10/1731565057371/Put/seqid=0 2024-11-14T06:17:38,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742183_1359 (size=12001) 2024-11-14T06:17:38,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:38,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57382 deadline: 1731565118878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:38,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:38,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:38,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57358 deadline: 1731565118878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:38,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:38,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:38,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:38,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:38,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:38,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:38,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57346 deadline: 1731565118887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:38,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:38,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1731565118887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:38,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:38,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:38,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:38,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:38,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:38,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:38,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:38,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57348 deadline: 1731565118887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:38,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:38,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:38,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:39,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:39,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57382 deadline: 1731565119197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:39,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:39,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57358 deadline: 1731565119197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:39,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:39,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:39,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:39,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:39,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:39,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:39,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:39,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57346 deadline: 1731565119207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:39,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:39,215 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:39,215 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:39,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1731565119207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:39,216 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:39,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:39,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:39,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:39,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:39,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57348 deadline: 1731565119207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:39,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:39,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:39,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:39,285 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/35915e179cf842ce97d461a72b584c2a 2024-11-14T06:17:39,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/2273c8cdaabf49118107593c6def0c72 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/2273c8cdaabf49118107593c6def0c72 2024-11-14T06:17:39,294 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/2273c8cdaabf49118107593c6def0c72, entries=150, sequenceid=50, filesize=30.2 K 2024-11-14T06:17:39,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/f8bd7fab0aca4689a95aebea33bfb425 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/f8bd7fab0aca4689a95aebea33bfb425 2024-11-14T06:17:39,297 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/f8bd7fab0aca4689a95aebea33bfb425, entries=150, sequenceid=50, filesize=11.7 K 2024-11-14T06:17:39,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/35915e179cf842ce97d461a72b584c2a as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/35915e179cf842ce97d461a72b584c2a 2024-11-14T06:17:39,302 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/35915e179cf842ce97d461a72b584c2a, entries=150, sequenceid=50, filesize=11.7 K 2024-11-14T06:17:39,304 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 8b0928b43a2d9ea8c8bf8f2250ec54df in 940ms, sequenceid=50, compaction requested=false 2024-11-14T06:17:39,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for 8b0928b43a2d9ea8c8bf8f2250ec54df: 2024-11-14T06:17:39,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:39,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-11-14T06:17:39,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-11-14T06:17:39,308 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-14T06:17:39,308 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0170 sec 2024-11-14T06:17:39,310 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 2.0240 sec 2024-11-14T06:17:39,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-14T06:17:39,437 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:17:39,438 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T06:17:39,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-14T06:17:39,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-14T06:17:39,442 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T06:17:39,443 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T06:17:39,443 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T06:17:39,538 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T06:17:39,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-14T06:17:39,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-14T06:17:39,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:39,596 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 8b0928b43a2d9ea8c8bf8f2250ec54df 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-14T06:17:39,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b0928b43a2d9ea8c8bf8f2250ec54df, store=A 2024-11-14T06:17:39,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:39,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b0928b43a2d9ea8c8bf8f2250ec54df, store=B 2024-11-14T06:17:39,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:39,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b0928b43a2d9ea8c8bf8f2250ec54df, store=C 2024-11-14T06:17:39,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:39,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411147a1c1cbbeea14c6ab7d0b689e4e9ac58_8b0928b43a2d9ea8c8bf8f2250ec54df is 50, key is test_row_0/A:col10/1731565058548/Put/seqid=0 2024-11-14T06:17:39,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742184_1360 (size=12154) 2024-11-14T06:17:39,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on 8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:39,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. as already flushing 2024-11-14T06:17:39,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:39,755 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731565059742, max timestamp=1731565059753Num uniques 859; 2024-11-14T06:17:39,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1731565119751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:39,755 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T06:17:39,756 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731565059742, max timestamp=1731565059753Num uniques 859; 2024-11-14T06:17:39,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:39,756 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T06:17:39,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:39,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:39,756 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731565059742, max timestamp=1731565059753Num uniques 859; 2024-11-14T06:17:39,756 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T06:17:39,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-14T06:17:39,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:39,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57346 deadline: 1731565119795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:39,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:39,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57358 deadline: 1731565119795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:39,807 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:39,807 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:39,807 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:39,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:39,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:39,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:39,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:39,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57348 deadline: 1731565119796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:39,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:39,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:39,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:39,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:39,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57382 deadline: 1731565119797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:39,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:39,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:39,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:39,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:39,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1731565119857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:39,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:39,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:39,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:39,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:39,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57346 deadline: 1731565119917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:39,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:39,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:39,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:39,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57358 deadline: 1731565119918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:39,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:39,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:39,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:39,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:39,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57348 deadline: 1731565119918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:39,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:39,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:39,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:39,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:39,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:39,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57382 deadline: 1731565119918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:39,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:39,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:39,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:40,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:40,011 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411147a1c1cbbeea14c6ab7d0b689e4e9ac58_8b0928b43a2d9ea8c8bf8f2250ec54df to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411147a1c1cbbeea14c6ab7d0b689e4e9ac58_8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:40,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/e322e2c634a84ffa9a29945bc9b01b1e, store: [table=TestAcidGuarantees family=A region=8b0928b43a2d9ea8c8bf8f2250ec54df] 2024-11-14T06:17:40,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/e322e2c634a84ffa9a29945bc9b01b1e is 175, key is test_row_0/A:col10/1731565058548/Put/seqid=0 2024-11-14T06:17:40,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742185_1361 (size=30955) 2024-11-14T06:17:40,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-14T06:17:40,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:40,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1731565120077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:40,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:40,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:40,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:40,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:40,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57346 deadline: 1731565120137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:40,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:40,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57358 deadline: 1731565120137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:40,150 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:40,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:40,150 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:40,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:40,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57348 deadline: 1731565120137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:40,150 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:40,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:40,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:40,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:40,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:40,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:40,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:40,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57382 deadline: 1731565120137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:40,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:40,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:40,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:40,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:40,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1731565120397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:40,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:40,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:40,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:40,416 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=61, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/e322e2c634a84ffa9a29945bc9b01b1e 2024-11-14T06:17:40,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/cd35d4671f1e425797703c530aa8cb7a is 50, key is test_row_0/B:col10/1731565058548/Put/seqid=0 2024-11-14T06:17:40,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742186_1362 (size=12001) 2024-11-14T06:17:40,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:40,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57346 deadline: 1731565120457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:40,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:40,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57358 deadline: 1731565120457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:40,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:40,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:40,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:40,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:40,464 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:40,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57348 deadline: 1731565120457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:40,464 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:40,464 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:40,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:40,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:40,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:40,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57382 deadline: 1731565120457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:40,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:40,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:40,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:40,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:40,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-14T06:17:40,828 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/cd35d4671f1e425797703c530aa8cb7a 2024-11-14T06:17:40,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/625d0c349d884ef8916b69c31da77a1c is 50, key is test_row_0/C:col10/1731565058548/Put/seqid=0 2024-11-14T06:17:40,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742187_1363 (size=12001) 2024-11-14T06:17:40,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:40,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1731565120917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:40,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:40,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:40,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:40,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:40,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57382 deadline: 1731565120967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:40,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:40,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:40,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:40,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:40,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57346 deadline: 1731565120977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:40,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:40,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:40,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57348 deadline: 1731565120977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:40,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:40,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:40,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:40,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:40,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57358 deadline: 1731565120978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:40,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:40,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:40,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:40,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:40,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:41,239 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/625d0c349d884ef8916b69c31da77a1c 2024-11-14T06:17:41,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/e322e2c634a84ffa9a29945bc9b01b1e as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/e322e2c634a84ffa9a29945bc9b01b1e 2024-11-14T06:17:41,246 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/e322e2c634a84ffa9a29945bc9b01b1e, entries=150, sequenceid=61, filesize=30.2 K 2024-11-14T06:17:41,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/cd35d4671f1e425797703c530aa8cb7a as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/cd35d4671f1e425797703c530aa8cb7a 2024-11-14T06:17:41,250 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/cd35d4671f1e425797703c530aa8cb7a, entries=150, sequenceid=61, filesize=11.7 K 2024-11-14T06:17:41,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/625d0c349d884ef8916b69c31da77a1c as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/625d0c349d884ef8916b69c31da77a1c 2024-11-14T06:17:41,254 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/625d0c349d884ef8916b69c31da77a1c, entries=150, sequenceid=61, filesize=11.7 K 2024-11-14T06:17:41,254 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 8b0928b43a2d9ea8c8bf8f2250ec54df in 1658ms, sequenceid=61, compaction requested=true 2024-11-14T06:17:41,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 8b0928b43a2d9ea8c8bf8f2250ec54df: 2024-11-14T06:17:41,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:41,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-14T06:17:41,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-14T06:17:41,257 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-14T06:17:41,257 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8130 sec 2024-11-14T06:17:41,259 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.8200 sec 2024-11-14T06:17:41,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-14T06:17:41,587 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:17:41,588 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T06:17:41,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-14T06:17:41,590 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T06:17:41,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-14T06:17:41,591 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T06:17:41,591 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T06:17:41,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-14T06:17:41,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-14T06:17:41,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:41,750 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 8b0928b43a2d9ea8c8bf8f2250ec54df 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-14T06:17:41,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b0928b43a2d9ea8c8bf8f2250ec54df, store=A 2024-11-14T06:17:41,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:17:41,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b0928b43a2d9ea8c8bf8f2250ec54df, store=B 2024-11-14T06:17:41,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:17:41,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b0928b43a2d9ea8c8bf8f2250ec54df, store=C 2024-11-14T06:17:41,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:17:41,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111433eb7d425e334b248f343496aa6fc994_8b0928b43a2d9ea8c8bf8f2250ec54df is 50, key is test_row_0/A:col10/1731565059791/Put/seqid=0 2024-11-14T06:17:41,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742188_1364 (size=12154) 2024-11-14T06:17:41,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-14T06:17:41,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on 8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:41,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. as already flushing 2024-11-14T06:17:41,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:41,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57382 deadline: 1731565121993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:41,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:41,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:41,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:41,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1731565121994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:41,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:41,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:42,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:42,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:42,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:42,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57346 deadline: 1731565121997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:42,007 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to ef93ef3a83c5,42283,1731564965945 for 'test_row_2' in 8b0928b43a2d9ea8c8bf8f2250ec54df of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T06:17:42,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:42,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:42,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:42,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:42,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57348 deadline: 1731565122007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:42,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:42,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57358 deadline: 1731565122007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:42,012 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to ef93ef3a83c5,42283,1731564965945 for 'test_row_0' in 8b0928b43a2d9ea8c8bf8f2250ec54df of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T06:17:42,013 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:42,013 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:42,013 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:42,013 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to ef93ef3a83c5,42283,1731564965945 for 'test_row_2' in 8b0928b43a2d9ea8c8bf8f2250ec54df of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T06:17:42,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:42,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:42,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:42,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:42,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57382 deadline: 1731565122107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:42,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:42,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:42,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:42,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:42,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1731565122107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:42,118 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:42,118 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:42,118 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:42,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:42,198 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111433eb7d425e334b248f343496aa6fc994_8b0928b43a2d9ea8c8bf8f2250ec54df to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111433eb7d425e334b248f343496aa6fc994_8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:42,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/539220dfdc0143119ec34ac676b14b8b, store: [table=TestAcidGuarantees family=A region=8b0928b43a2d9ea8c8bf8f2250ec54df] 2024-11-14T06:17:42,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/539220dfdc0143119ec34ac676b14b8b is 175, key is test_row_0/A:col10/1731565059791/Put/seqid=0 2024-11-14T06:17:42,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742189_1365 (size=30955) 2024-11-14T06:17:42,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-14T06:17:42,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:42,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57382 deadline: 1731565122327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:42,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:42,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1731565122327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:42,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:42,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:42,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:42,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:42,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:42,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:42,610 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=96, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/539220dfdc0143119ec34ac676b14b8b 2024-11-14T06:17:42,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/7fe0c4dff8914de99221dee4f7927e85 is 50, key is test_row_0/B:col10/1731565059791/Put/seqid=0 2024-11-14T06:17:42,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742190_1366 (size=12001) 2024-11-14T06:17:42,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:42,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57382 deadline: 1731565122647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:42,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:42,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1731565122647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:42,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:42,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:42,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:42,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:42,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:42,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:42,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-14T06:17:43,025 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/7fe0c4dff8914de99221dee4f7927e85 2024-11-14T06:17:43,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/1860b759719b4bc79f5d3d7346a4d497 is 50, key is test_row_0/C:col10/1731565059791/Put/seqid=0 2024-11-14T06:17:43,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742191_1367 (size=12001) 2024-11-14T06:17:43,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:43,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57382 deadline: 1731565123157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:43,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:43,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1731565123158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:43,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:43,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:43,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:43,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:43,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:43,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:43,435 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/1860b759719b4bc79f5d3d7346a4d497 2024-11-14T06:17:43,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/539220dfdc0143119ec34ac676b14b8b as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/539220dfdc0143119ec34ac676b14b8b 2024-11-14T06:17:43,445 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/539220dfdc0143119ec34ac676b14b8b, entries=150, sequenceid=96, filesize=30.2 K 2024-11-14T06:17:43,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/7fe0c4dff8914de99221dee4f7927e85 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/7fe0c4dff8914de99221dee4f7927e85 2024-11-14T06:17:43,450 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/7fe0c4dff8914de99221dee4f7927e85, entries=150, sequenceid=96, filesize=11.7 K 2024-11-14T06:17:43,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/1860b759719b4bc79f5d3d7346a4d497 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/1860b759719b4bc79f5d3d7346a4d497 2024-11-14T06:17:43,454 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/1860b759719b4bc79f5d3d7346a4d497, entries=150, sequenceid=96, filesize=11.7 K 2024-11-14T06:17:43,455 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 8b0928b43a2d9ea8c8bf8f2250ec54df in 1705ms, sequenceid=96, compaction requested=true 2024-11-14T06:17:43,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 8b0928b43a2d9ea8c8bf8f2250ec54df: 2024-11-14T06:17:43,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:43,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-14T06:17:43,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-14T06:17:43,458 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-14T06:17:43,458 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8660 sec 2024-11-14T06:17:43,459 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.8710 sec 2024-11-14T06:17:43,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-14T06:17:43,747 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:17:43,747 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T06:17:43,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-14T06:17:43,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-14T06:17:43,749 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T06:17:43,749 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T06:17:43,749 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T06:17:43,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-14T06:17:43,901 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-14T06:17:43,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:43,901 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 8b0928b43a2d9ea8c8bf8f2250ec54df 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-14T06:17:43,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b0928b43a2d9ea8c8bf8f2250ec54df, store=A 2024-11-14T06:17:43,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:43,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b0928b43a2d9ea8c8bf8f2250ec54df, store=B 2024-11-14T06:17:43,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:43,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b0928b43a2d9ea8c8bf8f2250ec54df, store=C 2024-11-14T06:17:43,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:43,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111473141ce1d49a403ca5bfba6849e50777_8b0928b43a2d9ea8c8bf8f2250ec54df is 50, key is test_row_0/A:col10/1731565061993/Put/seqid=0 2024-11-14T06:17:43,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742192_1368 (size=12154) 2024-11-14T06:17:44,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on 8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:44,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. as already flushing 2024-11-14T06:17:44,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-14T06:17:44,103 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731565064079, max timestamp=1731565064100Num uniques 859; 2024-11-14T06:17:44,103 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T06:17:44,103 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731565064079, max timestamp=1731565064100Num uniques 859; 2024-11-14T06:17:44,103 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731565064079, max timestamp=1731565064100Num uniques 859; 2024-11-14T06:17:44,103 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T06:17:44,103 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T06:17:44,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:44,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57348 deadline: 1731565124147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:44,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:44,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:44,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:44,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:44,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57346 deadline: 1731565124150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:44,163 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:44,163 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:44,163 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:44,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:44,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57358 deadline: 1731565124161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:44,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:44,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:44,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:44,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:44,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57382 deadline: 1731565124177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:44,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:44,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1731565124177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:44,191 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to ef93ef3a83c5,42283,1731564965945 for 'test_row_1' in 8b0928b43a2d9ea8c8bf8f2250ec54df of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T06:17:44,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:44,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:44,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:44,192 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to ef93ef3a83c5,42283,1731564965945 for 'test_row_1' in 8b0928b43a2d9ea8c8bf8f2250ec54df of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T06:17:44,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:44,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:44,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:44,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:44,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57348 deadline: 1731565124267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:44,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:44,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57346 deadline: 1731565124267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:44,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:44,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:44,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:44,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:44,275 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:44,275 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:44,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57358 deadline: 1731565124267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:44,275 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:44,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:44,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:44,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:44,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:44,316 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111473141ce1d49a403ca5bfba6849e50777_8b0928b43a2d9ea8c8bf8f2250ec54df to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111473141ce1d49a403ca5bfba6849e50777_8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:44,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/45a3933c9be9406e8c23e9c174fac46e, store: [table=TestAcidGuarantees family=A region=8b0928b43a2d9ea8c8bf8f2250ec54df] 2024-11-14T06:17:44,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/45a3933c9be9406e8c23e9c174fac46e is 175, key is test_row_0/A:col10/1731565061993/Put/seqid=0 2024-11-14T06:17:44,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742193_1369 (size=30955) 2024-11-14T06:17:44,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-14T06:17:44,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:44,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57348 deadline: 1731565124487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:44,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:44,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:44,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:44,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:44,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57346 deadline: 1731565124487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:44,497 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:44,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:44,497 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:44,497 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:44,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57358 deadline: 1731565124488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:44,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:44,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:44,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:44,720 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=107, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/45a3933c9be9406e8c23e9c174fac46e 2024-11-14T06:17:44,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/7c04561c234c444cb5eda7cc00b0d2d2 is 50, key is test_row_0/B:col10/1731565061993/Put/seqid=0 2024-11-14T06:17:44,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742194_1370 (size=12001) 2024-11-14T06:17:44,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:44,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57346 deadline: 1731565124807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:44,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:44,815 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:44,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57348 deadline: 1731565124807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:44,815 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:44,815 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:44,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:44,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:44,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:44,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:44,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57358 deadline: 1731565124808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:44,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:44,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:44,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:44,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-14T06:17:45,130 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/7c04561c234c444cb5eda7cc00b0d2d2 2024-11-14T06:17:45,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/92458d141a2042aa8031dea882bf3ec2 is 50, key is test_row_0/C:col10/1731565061993/Put/seqid=0 2024-11-14T06:17:45,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742195_1371 (size=12001) 2024-11-14T06:17:45,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:45,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57346 deadline: 1731565125327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:45,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:45,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:45,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57348 deadline: 1731565125327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:45,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:45,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:45,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:45,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:45,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:45,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57358 deadline: 1731565125327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:45,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:45,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:45,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:45,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:45,543 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/92458d141a2042aa8031dea882bf3ec2 2024-11-14T06:17:45,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/45a3933c9be9406e8c23e9c174fac46e as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/45a3933c9be9406e8c23e9c174fac46e 2024-11-14T06:17:45,553 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/45a3933c9be9406e8c23e9c174fac46e, entries=150, sequenceid=107, filesize=30.2 K 2024-11-14T06:17:45,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/7c04561c234c444cb5eda7cc00b0d2d2 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/7c04561c234c444cb5eda7cc00b0d2d2 2024-11-14T06:17:45,559 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/7c04561c234c444cb5eda7cc00b0d2d2, entries=150, sequenceid=107, filesize=11.7 K 2024-11-14T06:17:45,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/92458d141a2042aa8031dea882bf3ec2 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/92458d141a2042aa8031dea882bf3ec2 2024-11-14T06:17:45,566 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/92458d141a2042aa8031dea882bf3ec2, entries=150, sequenceid=107, filesize=11.7 K 2024-11-14T06:17:45,567 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 8b0928b43a2d9ea8c8bf8f2250ec54df in 1666ms, sequenceid=107, compaction requested=true 2024-11-14T06:17:45,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 8b0928b43a2d9ea8c8bf8f2250ec54df: 2024-11-14T06:17:45,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:45,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-14T06:17:45,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-14T06:17:45,570 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-14T06:17:45,570 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8200 sec 2024-11-14T06:17:45,572 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.8240 sec 2024-11-14T06:17:45,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-14T06:17:45,897 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:17:45,898 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T06:17:45,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-14T06:17:45,899 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T06:17:45,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-14T06:17:45,899 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T06:17:45,899 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T06:17:46,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-14T06:17:46,051 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-14T06:17:46,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:46,052 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 8b0928b43a2d9ea8c8bf8f2250ec54df 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-14T06:17:46,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b0928b43a2d9ea8c8bf8f2250ec54df, store=A 2024-11-14T06:17:46,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:17:46,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b0928b43a2d9ea8c8bf8f2250ec54df, store=B 2024-11-14T06:17:46,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:17:46,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b0928b43a2d9ea8c8bf8f2250ec54df, store=C 2024-11-14T06:17:46,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:17:46,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411140ea8750752a74a13b38ac2c3300b7160_8b0928b43a2d9ea8c8bf8f2250ec54df is 50, key is test_row_0/A:col10/1731565064160/Put/seqid=0 2024-11-14T06:17:46,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742196_1372 (size=12304) 2024-11-14T06:17:46,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-14T06:17:46,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on 8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:46,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. as already flushing 2024-11-14T06:17:46,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:46,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1731565126258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:46,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:46,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:46,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:46,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57382 deadline: 1731565126259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:46,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:46,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:46,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:46,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:46,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:46,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57346 deadline: 1731565126347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:46,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:46,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57348 deadline: 1731565126347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:46,349 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to ef93ef3a83c5,42283,1731564965945 for 'test_row_1' in 8b0928b43a2d9ea8c8bf8f2250ec54df of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T06:17:46,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:46,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:46,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:46,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:46,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57358 deadline: 1731565126347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:46,350 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to ef93ef3a83c5,42283,1731564965945 for 'test_row_1' in 8b0928b43a2d9ea8c8bf8f2250ec54df of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T06:17:46,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:46,350 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to ef93ef3a83c5,42283,1731564965945 for 'test_row_2' in 8b0928b43a2d9ea8c8bf8f2250ec54df of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T06:17:46,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:46,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:46,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:46,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:46,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:46,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:46,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1731565126377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:46,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:46,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:46,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:46,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57382 deadline: 1731565126377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:46,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:46,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:46,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:46,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:46,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:46,466 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411140ea8750752a74a13b38ac2c3300b7160_8b0928b43a2d9ea8c8bf8f2250ec54df to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411140ea8750752a74a13b38ac2c3300b7160_8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:46,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/09c201694a6c4593b7d8eec439dce808, store: [table=TestAcidGuarantees family=A region=8b0928b43a2d9ea8c8bf8f2250ec54df] 2024-11-14T06:17:46,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/09c201694a6c4593b7d8eec439dce808 is 175, key is test_row_0/A:col10/1731565064160/Put/seqid=0 2024-11-14T06:17:46,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742197_1373 (size=31105) 2024-11-14T06:17:46,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-14T06:17:46,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:46,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1731565126587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:46,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:46,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:46,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:46,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:46,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57382 deadline: 1731565126587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:46,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:46,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:46,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:46,870 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=142, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/09c201694a6c4593b7d8eec439dce808 2024-11-14T06:17:46,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/4bd1540165324bc99d42db9538746c13 is 50, key is test_row_0/B:col10/1731565064160/Put/seqid=0 2024-11-14T06:17:46,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742198_1374 (size=12151) 2024-11-14T06:17:46,897 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/4bd1540165324bc99d42db9538746c13 2024-11-14T06:17:46,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:46,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1731565126898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:46,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/7199b188f87b49c4b60ef80872478f12 is 50, key is test_row_0/C:col10/1731565064160/Put/seqid=0 2024-11-14T06:17:46,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:46,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:46,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:46,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57382 deadline: 1731565126898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:46,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:46,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:46,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:46,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:46,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742199_1375 (size=12151) 2024-11-14T06:17:47,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-14T06:17:47,321 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/7199b188f87b49c4b60ef80872478f12 2024-11-14T06:17:47,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/09c201694a6c4593b7d8eec439dce808 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/09c201694a6c4593b7d8eec439dce808 2024-11-14T06:17:47,332 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/09c201694a6c4593b7d8eec439dce808, entries=150, sequenceid=142, filesize=30.4 K 2024-11-14T06:17:47,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/4bd1540165324bc99d42db9538746c13 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/4bd1540165324bc99d42db9538746c13 2024-11-14T06:17:47,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,338 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/4bd1540165324bc99d42db9538746c13, entries=150, sequenceid=142, filesize=11.9 K 2024-11-14T06:17:47,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/7199b188f87b49c4b60ef80872478f12 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/7199b188f87b49c4b60ef80872478f12 2024-11-14T06:17:47,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,343 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/7199b188f87b49c4b60ef80872478f12, entries=150, sequenceid=142, filesize=11.9 K 2024-11-14T06:17:47,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,343 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 8b0928b43a2d9ea8c8bf8f2250ec54df in 1292ms, sequenceid=142, compaction requested=true 2024-11-14T06:17:47,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 8b0928b43a2d9ea8c8bf8f2250ec54df: 2024-11-14T06:17:47,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:47,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-14T06:17:47,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-14T06:17:47,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,347 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-14T06:17:47,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,347 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4460 sec 2024-11-14T06:17:47,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,349 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.4490 sec 2024-11-14T06:17:47,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on 8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:47,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,428 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8b0928b43a2d9ea8c8bf8f2250ec54df 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-14T06:17:47,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b0928b43a2d9ea8c8bf8f2250ec54df, store=A 2024-11-14T06:17:47,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:47,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b0928b43a2d9ea8c8bf8f2250ec54df, store=B 2024-11-14T06:17:47,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:47,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b0928b43a2d9ea8c8bf8f2250ec54df, store=C 2024-11-14T06:17:47,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:47,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411146931776b164d406fb5b5c8c50418d8c7_8b0928b43a2d9ea8c8bf8f2250ec54df is 50, key is test_row_1/A:col10/1731565067427/Put/seqid=0 2024-11-14T06:17:47,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742200_1376 (size=12304) 2024-11-14T06:17:47,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,446 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,450 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411146931776b164d406fb5b5c8c50418d8c7_8b0928b43a2d9ea8c8bf8f2250ec54df to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411146931776b164d406fb5b5c8c50418d8c7_8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:47,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,450 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/7c8477cd734b43c3b33634608b2a4b03, store: [table=TestAcidGuarantees family=A region=8b0928b43a2d9ea8c8bf8f2250ec54df] 2024-11-14T06:17:47,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/7c8477cd734b43c3b33634608b2a4b03 is 175, key is test_row_1/A:col10/1731565067427/Put/seqid=0 2024-11-14T06:17:47,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742201_1377 (size=31101) 2024-11-14T06:17:47,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:47,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:47,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57382 deadline: 1731565127565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:47,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:47,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:47,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:47,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:47,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1731565127566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:47,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:47,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:47,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:47,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:47,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57382 deadline: 1731565127677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:47,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:47,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:47,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:47,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:47,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1731565127687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:47,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:47,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:47,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:47,869 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=154, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/7c8477cd734b43c3b33634608b2a4b03 2024-11-14T06:17:47,877 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/0301544542c04e80b9af9a72d37d278d is 50, key is test_row_1/B:col10/1731565067427/Put/seqid=0 2024-11-14T06:17:47,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:47,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57382 deadline: 1731565127889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:47,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:47,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:47,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:47,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742202_1378 (size=9757) 2024-11-14T06:17:47,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:47,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1731565127907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:47,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:47,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:47,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:48,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-14T06:17:48,047 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:17:48,048 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T06:17:48,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-14T06:17:48,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-14T06:17:48,049 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T06:17:48,050 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T06:17:48,050 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T06:17:48,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-14T06:17:48,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-14T06:17:48,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:48,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. as already flushing 2024-11-14T06:17:48,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:48,202 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:48,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:48,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:48,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:48,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57382 deadline: 1731565128207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:48,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:48,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:48,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:48,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:48,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1731565128217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:48,223 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:48,223 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:48,223 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:48,301 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/0301544542c04e80b9af9a72d37d278d 2024-11-14T06:17:48,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/deaf8ce985cf420284f561361b8e2c11 is 50, key is test_row_1/C:col10/1731565067427/Put/seqid=0 2024-11-14T06:17:48,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742203_1379 (size=9757) 2024-11-14T06:17:48,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-14T06:17:48,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:48,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. as already flushing 2024-11-14T06:17:48,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:48,355 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:48,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:48,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:48,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:48,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57348 deadline: 1731565128357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:48,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:48,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57346 deadline: 1731565128357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:48,364 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to ef93ef3a83c5,42283,1731564965945 for 'test_row_1' in 8b0928b43a2d9ea8c8bf8f2250ec54df of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T06:17:48,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:48,364 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to ef93ef3a83c5,42283,1731564965945 for 'test_row_1' in 8b0928b43a2d9ea8c8bf8f2250ec54df of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T06:17:48,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:48,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:48,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:48,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:48,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:48,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-14T06:17:48,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:48,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57358 deadline: 1731565128367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:48,380 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to ef93ef3a83c5,42283,1731564965945 for 'test_row_2' in 8b0928b43a2d9ea8c8bf8f2250ec54df of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T06:17:48,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:48,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:48,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:48,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-14T06:17:48,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:48,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. as already flushing 2024-11-14T06:17:48,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:48,507 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:48,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:48,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:48,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-14T06:17:48,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:48,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. as already flushing 2024-11-14T06:17:48,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:48,660 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:48,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:48,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:48,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-14T06:17:48,709 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/deaf8ce985cf420284f561361b8e2c11 2024-11-14T06:17:48,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/7c8477cd734b43c3b33634608b2a4b03 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/7c8477cd734b43c3b33634608b2a4b03 2024-11-14T06:17:48,716 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/7c8477cd734b43c3b33634608b2a4b03, entries=150, sequenceid=154, filesize=30.4 K 2024-11-14T06:17:48,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/0301544542c04e80b9af9a72d37d278d as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/0301544542c04e80b9af9a72d37d278d 2024-11-14T06:17:48,719 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/0301544542c04e80b9af9a72d37d278d, entries=100, sequenceid=154, filesize=9.5 K 2024-11-14T06:17:48,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/deaf8ce985cf420284f561361b8e2c11 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/deaf8ce985cf420284f561361b8e2c11 2024-11-14T06:17:48,722 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/deaf8ce985cf420284f561361b8e2c11, entries=100, sequenceid=154, filesize=9.5 K 2024-11-14T06:17:48,723 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 8b0928b43a2d9ea8c8bf8f2250ec54df in 1295ms, sequenceid=154, compaction requested=true 2024-11-14T06:17:48,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8b0928b43a2d9ea8c8bf8f2250ec54df: 2024-11-14T06:17:48,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b0928b43a2d9ea8c8bf8f2250ec54df:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T06:17:48,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:48,723 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-11-14T06:17:48,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b0928b43a2d9ea8c8bf8f2250ec54df:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T06:17:48,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:17:48,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b0928b43a2d9ea8c8bf8f2250ec54df:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T06:17:48,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T06:17:48,724 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-11-14T06:17:48,728 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 208387 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-11-14T06:17:48,728 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 79569 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-11-14T06:17:48,728 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): 8b0928b43a2d9ea8c8bf8f2250ec54df/B is initiating minor compaction (all files) 2024-11-14T06:17:48,728 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): 8b0928b43a2d9ea8c8bf8f2250ec54df/A is initiating minor compaction (all files) 2024-11-14T06:17:48,728 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b0928b43a2d9ea8c8bf8f2250ec54df/A in TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:48,729 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/2aa41640c78144a38b1dba9f2f406934, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/2273c8cdaabf49118107593c6def0c72, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/e322e2c634a84ffa9a29945bc9b01b1e, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/539220dfdc0143119ec34ac676b14b8b, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/45a3933c9be9406e8c23e9c174fac46e, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/09c201694a6c4593b7d8eec439dce808, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/7c8477cd734b43c3b33634608b2a4b03] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp, totalSize=203.5 K 2024-11-14T06:17:48,729 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=9 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:48,729 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. files: [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/2aa41640c78144a38b1dba9f2f406934, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/2273c8cdaabf49118107593c6def0c72, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/e322e2c634a84ffa9a29945bc9b01b1e, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/539220dfdc0143119ec34ac676b14b8b, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/45a3933c9be9406e8c23e9c174fac46e, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/09c201694a6c4593b7d8eec439dce808, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/7c8477cd734b43c3b33634608b2a4b03] 2024-11-14T06:17:48,729 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b0928b43a2d9ea8c8bf8f2250ec54df/B in TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:48,729 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/35d1c9ef29eb43df96016ebc9e8308a4, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/f8bd7fab0aca4689a95aebea33bfb425, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/cd35d4671f1e425797703c530aa8cb7a, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/7fe0c4dff8914de99221dee4f7927e85, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/7c04561c234c444cb5eda7cc00b0d2d2, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/4bd1540165324bc99d42db9538746c13, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/0301544542c04e80b9af9a72d37d278d] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp, totalSize=77.7 K 2024-11-14T06:17:48,729 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2aa41640c78144a38b1dba9f2f406934, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731565057297 2024-11-14T06:17:48,729 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2273c8cdaabf49118107593c6def0c72, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731565057371 2024-11-14T06:17:48,729 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 35d1c9ef29eb43df96016ebc9e8308a4, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731565057297 2024-11-14T06:17:48,730 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting f8bd7fab0aca4689a95aebea33bfb425, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731565057371 2024-11-14T06:17:48,730 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting e322e2c634a84ffa9a29945bc9b01b1e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731565058546 2024-11-14T06:17:48,730 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 539220dfdc0143119ec34ac676b14b8b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731565059791 2024-11-14T06:17:48,730 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting cd35d4671f1e425797703c530aa8cb7a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731565058546 2024-11-14T06:17:48,730 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45a3933c9be9406e8c23e9c174fac46e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731565061986 2024-11-14T06:17:48,730 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fe0c4dff8914de99221dee4f7927e85, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731565059791 2024-11-14T06:17:48,730 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09c201694a6c4593b7d8eec439dce808, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731565064144 2024-11-14T06:17:48,730 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c04561c234c444cb5eda7cc00b0d2d2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731565061986 2024-11-14T06:17:48,730 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c8477cd734b43c3b33634608b2a4b03, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731565066257 2024-11-14T06:17:48,731 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bd1540165324bc99d42db9538746c13, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731565064144 2024-11-14T06:17:48,731 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 0301544542c04e80b9af9a72d37d278d, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731565066258 2024-11-14T06:17:48,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on 8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:48,738 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731565067503, max timestamp=1731565068735Num uniques 859; 2024-11-14T06:17:48,739 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731565067503, max timestamp=1731565068735Num uniques 859; 2024-11-14T06:17:48,739 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T06:17:48,739 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T06:17:48,739 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731565067503, max timestamp=1731565068735Num uniques 859; 2024-11-14T06:17:48,739 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8b0928b43a2d9ea8c8bf8f2250ec54df 3/3 column families, dataSize=58.14 KB heapSize=164.17 KB 2024-11-14T06:17:48,739 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T06:17:48,740 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8b0928b43a2d9ea8c8bf8f2250ec54df] 2024-11-14T06:17:48,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b0928b43a2d9ea8c8bf8f2250ec54df, store=A 2024-11-14T06:17:48,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:17:48,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b0928b43a2d9ea8c8bf8f2250ec54df, store=B 2024-11-14T06:17:48,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:17:48,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b0928b43a2d9ea8c8bf8f2250ec54df, store=C 2024-11-14T06:17:48,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:17:48,742 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114383d562f3b9d4789abf3a8189992d068_8b0928b43a2d9ea8c8bf8f2250ec54df store=[table=TestAcidGuarantees family=A region=8b0928b43a2d9ea8c8bf8f2250ec54df] 2024-11-14T06:17:48,744 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b0928b43a2d9ea8c8bf8f2250ec54df#B#compaction#325 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:17:48,745 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/7b6be02cd67745608859af71a22068ca is 50, key is test_row_0/B:col10/1731565064160/Put/seqid=0 2024-11-14T06:17:48,746 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114383d562f3b9d4789abf3a8189992d068_8b0928b43a2d9ea8c8bf8f2250ec54df, store=[table=TestAcidGuarantees family=A region=8b0928b43a2d9ea8c8bf8f2250ec54df] 2024-11-14T06:17:48,746 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114383d562f3b9d4789abf3a8189992d068_8b0928b43a2d9ea8c8bf8f2250ec54df because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8b0928b43a2d9ea8c8bf8f2250ec54df] 2024-11-14T06:17:48,762 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114fd8187bfe26d480491ea1d285e05b535_8b0928b43a2d9ea8c8bf8f2250ec54df is 50, key is test_row_0/A:col10/1731565067564/Put/seqid=0 2024-11-14T06:17:48,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742205_1381 (size=12391) 2024-11-14T06:17:48,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742206_1382 (size=17284) 2024-11-14T06:17:48,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742204_1380 (size=4469) 2024-11-14T06:17:48,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-14T06:17:48,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:48,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. as already flushing 2024-11-14T06:17:48,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:48,813 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:48,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:48,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:48,863 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731565068811, max timestamp=1731565068861Num uniques 859; 2024-11-14T06:17:48,863 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731565068811, max timestamp=1731565068861Num uniques 859; 2024-11-14T06:17:48,864 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731565068811, max timestamp=1731565068861Num uniques 859; 2024-11-14T06:17:48,864 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T06:17:48,864 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T06:17:48,864 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T06:17:48,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-14T06:17:48,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:48,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. as already flushing 2024-11-14T06:17:48,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:48,966 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:48,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:48,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:48,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731565068947, max timestamp=1731565068984Num uniques 1068; 2024-11-14T06:17:48,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-14T06:17:48,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731565068947, max timestamp=1731565068984Num uniques 1068; 2024-11-14T06:17:48,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-14T06:17:48,989 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731565068947, max timestamp=1731565068984Num uniques 1068; 2024-11-14T06:17:48,989 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-14T06:17:49,119 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731565069069, max timestamp=1731565069115Num uniques 1077; 2024-11-14T06:17:49,119 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-14T06:17:49,119 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731565069069, max timestamp=1731565069115Num uniques 1077; 2024-11-14T06:17:49,119 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-14T06:17:49,119 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731565069069, max timestamp=1731565069115Num uniques 1077; 2024-11-14T06:17:49,119 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-14T06:17:49,125 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-14T06:17:49,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:49,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. as already flushing 2024-11-14T06:17:49,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:49,125 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:49,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:49,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:49,166 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:49,166 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/7b6be02cd67745608859af71a22068ca as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/7b6be02cd67745608859af71a22068ca 2024-11-14T06:17:49,166 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b0928b43a2d9ea8c8bf8f2250ec54df#A#compaction#324 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:17:49,167 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/6cf9815e8cc3400ca646696169265d28 is 175, key is test_row_0/A:col10/1731565064160/Put/seqid=0 2024-11-14T06:17:49,171 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114fd8187bfe26d480491ea1d285e05b535_8b0928b43a2d9ea8c8bf8f2250ec54df to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114fd8187bfe26d480491ea1d285e05b535_8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:49,178 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/c753f9ecab0f4746a52328198c1bdeb5, store: [table=TestAcidGuarantees family=A region=8b0928b43a2d9ea8c8bf8f2250ec54df] 2024-11-14T06:17:49,178 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/c753f9ecab0f4746a52328198c1bdeb5 is 175, key is test_row_0/A:col10/1731565067564/Put/seqid=0 2024-11-14T06:17:49,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742207_1383 (size=31452) 2024-11-14T06:17:49,181 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in 8b0928b43a2d9ea8c8bf8f2250ec54df/B of 8b0928b43a2d9ea8c8bf8f2250ec54df into 7b6be02cd67745608859af71a22068ca(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:17:49,181 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b0928b43a2d9ea8c8bf8f2250ec54df: 2024-11-14T06:17:49,181 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., storeName=8b0928b43a2d9ea8c8bf8f2250ec54df/B, priority=9, startTime=1731565068723; duration=0sec 2024-11-14T06:17:49,182 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:17:49,182 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b0928b43a2d9ea8c8bf8f2250ec54df:B 2024-11-14T06:17:49,182 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-11-14T06:17:49,185 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 79569 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-11-14T06:17:49,185 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): 8b0928b43a2d9ea8c8bf8f2250ec54df/C is initiating minor compaction (all files) 2024-11-14T06:17:49,185 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b0928b43a2d9ea8c8bf8f2250ec54df/C in TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:49,186 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/4d370e25165a4225b2e2b770dc5cba02, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/35915e179cf842ce97d461a72b584c2a, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/625d0c349d884ef8916b69c31da77a1c, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/1860b759719b4bc79f5d3d7346a4d497, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/92458d141a2042aa8031dea882bf3ec2, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/7199b188f87b49c4b60ef80872478f12, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/deaf8ce985cf420284f561361b8e2c11] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp, totalSize=77.7 K 2024-11-14T06:17:49,186 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d370e25165a4225b2e2b770dc5cba02, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731565057297 2024-11-14T06:17:49,186 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 35915e179cf842ce97d461a72b584c2a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731565057371 2024-11-14T06:17:49,187 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 625d0c349d884ef8916b69c31da77a1c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731565058546 2024-11-14T06:17:49,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-14T06:17:49,188 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 1860b759719b4bc79f5d3d7346a4d497, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731565059791 2024-11-14T06:17:49,188 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/6cf9815e8cc3400ca646696169265d28 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/6cf9815e8cc3400ca646696169265d28 2024-11-14T06:17:49,188 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 92458d141a2042aa8031dea882bf3ec2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731565061986 2024-11-14T06:17:49,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742208_1384 (size=48389) 2024-11-14T06:17:49,189 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 7199b188f87b49c4b60ef80872478f12, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731565064144 2024-11-14T06:17:49,189 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting deaf8ce985cf420284f561361b8e2c11, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731565066258 2024-11-14T06:17:49,190 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=176, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/c753f9ecab0f4746a52328198c1bdeb5 2024-11-14T06:17:49,193 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in 8b0928b43a2d9ea8c8bf8f2250ec54df/A of 8b0928b43a2d9ea8c8bf8f2250ec54df into 6cf9815e8cc3400ca646696169265d28(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:17:49,193 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b0928b43a2d9ea8c8bf8f2250ec54df: 2024-11-14T06:17:49,193 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., storeName=8b0928b43a2d9ea8c8bf8f2250ec54df/A, priority=9, startTime=1731565068723; duration=0sec 2024-11-14T06:17:49,193 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:49,193 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b0928b43a2d9ea8c8bf8f2250ec54df:A 2024-11-14T06:17:49,200 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/08030e0610ea4d8ebd4e70c1b86a274c is 50, key is test_row_0/B:col10/1731565067564/Put/seqid=0 2024-11-14T06:17:49,203 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b0928b43a2d9ea8c8bf8f2250ec54df#C#compaction#328 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:17:49,204 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/ff1f0d33941d41d5aa00da668029d4d0 is 50, key is test_row_0/C:col10/1731565064160/Put/seqid=0 2024-11-14T06:17:49,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742209_1385 (size=12151) 2024-11-14T06:17:49,218 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/08030e0610ea4d8ebd4e70c1b86a274c 2024-11-14T06:17:49,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/2fb8fb99f2da46a58a0da12ec59f557d is 50, key is test_row_0/C:col10/1731565067564/Put/seqid=0 2024-11-14T06:17:49,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742210_1386 (size=12391) 2024-11-14T06:17:49,229 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/ff1f0d33941d41d5aa00da668029d4d0 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/ff1f0d33941d41d5aa00da668029d4d0 2024-11-14T06:17:49,234 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in 8b0928b43a2d9ea8c8bf8f2250ec54df/C of 8b0928b43a2d9ea8c8bf8f2250ec54df into ff1f0d33941d41d5aa00da668029d4d0(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:17:49,234 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b0928b43a2d9ea8c8bf8f2250ec54df: 2024-11-14T06:17:49,234 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., storeName=8b0928b43a2d9ea8c8bf8f2250ec54df/C, priority=9, startTime=1731565068723; duration=0sec 2024-11-14T06:17:49,234 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:49,234 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b0928b43a2d9ea8c8bf8f2250ec54df:C 2024-11-14T06:17:49,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742211_1387 (size=12151) 2024-11-14T06:17:49,254 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1731565069214, max timestamp=1731565069250Num uniques 1086; 2024-11-14T06:17:49,254 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1731565069214, max timestamp=1731565069250Num uniques 1086; 2024-11-14T06:17:49,254 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=15386, suffix heap size=140912,new segment heap size=35184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=336 2024-11-14T06:17:49,254 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=15386, suffix heap size=140912,new segment heap size=35184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=336 2024-11-14T06:17:49,254 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1731565069214, max timestamp=1731565069250Num uniques 1086; 2024-11-14T06:17:49,254 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=15386, suffix heap size=140912,new segment heap size=35184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=336 2024-11-14T06:17:49,277 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-14T06:17:49,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:49,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. as already flushing 2024-11-14T06:17:49,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:49,278 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:49,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:49,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:49,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:49,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1731565129328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:49,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:49,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:49,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:49,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57382 deadline: 1731565129335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:49,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:49,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:49,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:49,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:49,430 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-14T06:17:49,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:49,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. as already flushing 2024-11-14T06:17:49,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:49,430 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:49,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:49,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:49,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:49,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1731565129447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:49,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:49,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:49,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57382 deadline: 1731565129447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:49,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:49,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:49,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:49,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:49,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:49,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-14T06:17:49,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:49,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. as already flushing 2024-11-14T06:17:49,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:49,583 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:49,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:49,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:49,637 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/2fb8fb99f2da46a58a0da12ec59f557d 2024-11-14T06:17:49,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/c753f9ecab0f4746a52328198c1bdeb5 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/c753f9ecab0f4746a52328198c1bdeb5 2024-11-14T06:17:49,647 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/c753f9ecab0f4746a52328198c1bdeb5, entries=250, sequenceid=176, filesize=47.3 K 2024-11-14T06:17:49,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/08030e0610ea4d8ebd4e70c1b86a274c as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/08030e0610ea4d8ebd4e70c1b86a274c 2024-11-14T06:17:49,651 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/08030e0610ea4d8ebd4e70c1b86a274c, entries=150, sequenceid=176, filesize=11.9 K 2024-11-14T06:17:49,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/2fb8fb99f2da46a58a0da12ec59f557d as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/2fb8fb99f2da46a58a0da12ec59f557d 2024-11-14T06:17:49,655 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/2fb8fb99f2da46a58a0da12ec59f557d, entries=150, sequenceid=176, filesize=11.9 K 2024-11-14T06:17:49,656 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=154.31 KB/158010 for 8b0928b43a2d9ea8c8bf8f2250ec54df in 917ms, sequenceid=176, compaction requested=false 2024-11-14T06:17:49,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8b0928b43a2d9ea8c8bf8f2250ec54df: 2024-11-14T06:17:49,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on 8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:49,668 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731565069325, max timestamp=1731565069664Num uniques 1195; 2024-11-14T06:17:49,668 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=54730, new segment data size=8930, suffix heap size=152248,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1195, new segment cells count=195 2024-11-14T06:17:49,668 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8b0928b43a2d9ea8c8bf8f2250ec54df 3/3 column families, dataSize=116.29 KB heapSize=320.14 KB 2024-11-14T06:17:49,669 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731565069325, max timestamp=1731565069664Num uniques 1195; 2024-11-14T06:17:49,669 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731565069325, max timestamp=1731565069664Num uniques 1195; 2024-11-14T06:17:49,669 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=54730, new segment data size=8930, suffix heap size=152248,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1195, new segment cells count=195 2024-11-14T06:17:49,669 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=54730, new segment data size=8930, suffix heap size=152248,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1195, new segment cells count=195 2024-11-14T06:17:49,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b0928b43a2d9ea8c8bf8f2250ec54df, store=A 2024-11-14T06:17:49,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:17:49,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b0928b43a2d9ea8c8bf8f2250ec54df, store=B 2024-11-14T06:17:49,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:17:49,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b0928b43a2d9ea8c8bf8f2250ec54df, store=C 2024-11-14T06:17:49,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:17:49,675 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411144ff1ef5c1fa342e3bf727c3e7d80ccbb_8b0928b43a2d9ea8c8bf8f2250ec54df is 50, key is test_row_0/A:col10/1731565069666/Put/seqid=0 2024-11-14T06:17:49,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742212_1388 (size=17534) 2024-11-14T06:17:49,695 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:49,698 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411144ff1ef5c1fa342e3bf727c3e7d80ccbb_8b0928b43a2d9ea8c8bf8f2250ec54df to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144ff1ef5c1fa342e3bf727c3e7d80ccbb_8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:49,699 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/64e8a2b69e1345cbbe95622b38cb4419, store: [table=TestAcidGuarantees family=A region=8b0928b43a2d9ea8c8bf8f2250ec54df] 2024-11-14T06:17:49,700 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/64e8a2b69e1345cbbe95622b38cb4419 is 175, key is test_row_0/A:col10/1731565069666/Put/seqid=0 2024-11-14T06:17:49,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742213_1389 (size=48639) 2024-11-14T06:17:49,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-14T06:17:49,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:49,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. as already flushing 2024-11-14T06:17:49,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:49,735 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:49,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:49,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:49,756 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731565069678, max timestamp=1731565069753Num uniques 859; 2024-11-14T06:17:49,756 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731565069678, max timestamp=1731565069753Num uniques 859; 2024-11-14T06:17:49,756 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T06:17:49,756 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T06:17:49,756 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731565069678, max timestamp=1731565069753Num uniques 859; 2024-11-14T06:17:49,756 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T06:17:49,864 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731565069808, max timestamp=1731565069860Num uniques 1068; 2024-11-14T06:17:49,864 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731565069808, max timestamp=1731565069860Num uniques 1068; 2024-11-14T06:17:49,864 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-14T06:17:49,865 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-14T06:17:49,866 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731565069808, max timestamp=1731565069860Num uniques 1068; 2024-11-14T06:17:49,866 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-14T06:17:49,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-14T06:17:49,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:49,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. as already flushing 2024-11-14T06:17:49,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:49,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:49,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:49,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:49,955 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731565069889, max timestamp=1731565069952Num uniques 1077; 2024-11-14T06:17:49,955 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-14T06:17:49,955 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731565069889, max timestamp=1731565069952Num uniques 1077; 2024-11-14T06:17:49,956 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-14T06:17:49,956 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731565069889, max timestamp=1731565069952Num uniques 1077; 2024-11-14T06:17:49,956 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-14T06:17:50,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-14T06:17:50,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:50,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. as already flushing 2024-11-14T06:17:50,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:50,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:50,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:50,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:50,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731565070050, max timestamp=1731565070100Num uniques 1086; 2024-11-14T06:17:50,105 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731565070050, max timestamp=1731565070100Num uniques 1086; 2024-11-14T06:17:50,105 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-14T06:17:50,105 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-14T06:17:50,106 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=269, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/64e8a2b69e1345cbbe95622b38cb4419 2024-11-14T06:17:50,107 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731565070050, max timestamp=1731565070100Num uniques 1086; 2024-11-14T06:17:50,107 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-14T06:17:50,113 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/b03e6fd736174f0f868fe2d17133b9ec is 50, key is test_row_0/B:col10/1731565069666/Put/seqid=0 2024-11-14T06:17:50,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742214_1390 (size=12301) 2024-11-14T06:17:50,132 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/b03e6fd736174f0f868fe2d17133b9ec 2024-11-14T06:17:50,138 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/1576fb66e1fb4fb9b167c21a7aaa58cb is 50, key is test_row_0/C:col10/1731565069666/Put/seqid=0 2024-11-14T06:17:50,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742215_1391 (size=12301) 2024-11-14T06:17:50,171 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/1576fb66e1fb4fb9b167c21a7aaa58cb 2024-11-14T06:17:50,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/64e8a2b69e1345cbbe95622b38cb4419 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/64e8a2b69e1345cbbe95622b38cb4419 2024-11-14T06:17:50,179 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/64e8a2b69e1345cbbe95622b38cb4419, entries=250, sequenceid=269, filesize=47.5 K 2024-11-14T06:17:50,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/b03e6fd736174f0f868fe2d17133b9ec as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/b03e6fd736174f0f868fe2d17133b9ec 2024-11-14T06:17:50,188 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/b03e6fd736174f0f868fe2d17133b9ec, entries=150, sequenceid=269, filesize=12.0 K 2024-11-14T06:17:50,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/1576fb66e1fb4fb9b167c21a7aaa58cb as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/1576fb66e1fb4fb9b167c21a7aaa58cb 2024-11-14T06:17:50,193 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/1576fb66e1fb4fb9b167c21a7aaa58cb, entries=150, sequenceid=269, filesize=12.0 K 2024-11-14T06:17:50,194 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~82.73 KB/84720, currentSize=120.76 KB/123660 for 8b0928b43a2d9ea8c8bf8f2250ec54df in 526ms, sequenceid=269, compaction requested=true 2024-11-14T06:17:50,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8b0928b43a2d9ea8c8bf8f2250ec54df: 2024-11-14T06:17:50,194 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:17:50,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b0928b43a2d9ea8c8bf8f2250ec54df:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T06:17:50,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:50,194 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:17:50,194 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-14T06:17:50,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:50,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b0928b43a2d9ea8c8bf8f2250ec54df:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T06:17:50,195 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 8b0928b43a2d9ea8c8bf8f2250ec54df 3/3 column families, dataSize=120.76 KB heapSize=337.92 KB 2024-11-14T06:17:50,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:50,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b0928b43a2d9ea8c8bf8f2250ec54df:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T06:17:50,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:17:50,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b0928b43a2d9ea8c8bf8f2250ec54df, store=A 2024-11-14T06:17:50,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:17:50,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b0928b43a2d9ea8c8bf8f2250ec54df, store=B 2024-11-14T06:17:50,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:17:50,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b0928b43a2d9ea8c8bf8f2250ec54df, store=C 2024-11-14T06:17:50,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:17:50,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-14T06:17:50,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on 8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:50,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. as already flushing 2024-11-14T06:17:50,198 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36843 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:17:50,198 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): 8b0928b43a2d9ea8c8bf8f2250ec54df/B is initiating minor compaction (all files) 2024-11-14T06:17:50,198 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b0928b43a2d9ea8c8bf8f2250ec54df/B in TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:50,199 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/7b6be02cd67745608859af71a22068ca, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/08030e0610ea4d8ebd4e70c1b86a274c, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/b03e6fd736174f0f868fe2d17133b9ec] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp, totalSize=36.0 K 2024-11-14T06:17:50,199 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 128480 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:17:50,199 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): 8b0928b43a2d9ea8c8bf8f2250ec54df/A is initiating minor compaction (all files) 2024-11-14T06:17:50,199 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b0928b43a2d9ea8c8bf8f2250ec54df/A in TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:50,199 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/6cf9815e8cc3400ca646696169265d28, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/c753f9ecab0f4746a52328198c1bdeb5, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/64e8a2b69e1345cbbe95622b38cb4419] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp, totalSize=125.5 K 2024-11-14T06:17:50,199 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:50,199 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. files: [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/6cf9815e8cc3400ca646696169265d28, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/c753f9ecab0f4746a52328198c1bdeb5, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/64e8a2b69e1345cbbe95622b38cb4419] 2024-11-14T06:17:50,199 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6cf9815e8cc3400ca646696169265d28, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731565064160 2024-11-14T06:17:50,199 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b6be02cd67745608859af71a22068ca, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731565064160 2024-11-14T06:17:50,202 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 08030e0610ea4d8ebd4e70c1b86a274c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731565067503 2024-11-14T06:17:50,202 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting c753f9ecab0f4746a52328198c1bdeb5, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731565067503 2024-11-14T06:17:50,203 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting b03e6fd736174f0f868fe2d17133b9ec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1731565069325 2024-11-14T06:17:50,203 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64e8a2b69e1345cbbe95622b38cb4419, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1731565069325 2024-11-14T06:17:50,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114563a9a1c40c942968c8be1dd3dbbc00a_8b0928b43a2d9ea8c8bf8f2250ec54df is 50, key is test_row_0/A:col10/1731565070164/Put/seqid=0 2024-11-14T06:17:50,222 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b0928b43a2d9ea8c8bf8f2250ec54df#B#compaction#334 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:17:50,223 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/05d91ff6c48348c8be5a959a1a28ccbf is 50, key is test_row_0/B:col10/1731565069666/Put/seqid=0 2024-11-14T06:17:50,225 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8b0928b43a2d9ea8c8bf8f2250ec54df] 2024-11-14T06:17:50,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742216_1392 (size=12643) 2024-11-14T06:17:50,237 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411145529e70796084bb2b184a5d5a4493260_8b0928b43a2d9ea8c8bf8f2250ec54df store=[table=TestAcidGuarantees family=A region=8b0928b43a2d9ea8c8bf8f2250ec54df] 2024-11-14T06:17:50,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742217_1393 (size=17534) 2024-11-14T06:17:50,239 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411145529e70796084bb2b184a5d5a4493260_8b0928b43a2d9ea8c8bf8f2250ec54df, store=[table=TestAcidGuarantees family=A region=8b0928b43a2d9ea8c8bf8f2250ec54df] 2024-11-14T06:17:50,239 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411145529e70796084bb2b184a5d5a4493260_8b0928b43a2d9ea8c8bf8f2250ec54df because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8b0928b43a2d9ea8c8bf8f2250ec54df] 2024-11-14T06:17:50,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:50,248 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114563a9a1c40c942968c8be1dd3dbbc00a_8b0928b43a2d9ea8c8bf8f2250ec54df to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114563a9a1c40c942968c8be1dd3dbbc00a_8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:50,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/57eca7bc2fdd4599be2dd308ce17083f, store: [table=TestAcidGuarantees family=A region=8b0928b43a2d9ea8c8bf8f2250ec54df] 2024-11-14T06:17:50,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/57eca7bc2fdd4599be2dd308ce17083f is 175, key is test_row_0/A:col10/1731565070164/Put/seqid=0 2024-11-14T06:17:50,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:50,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1731565130245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:50,258 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:50,258 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:50,258 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:50,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:50,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57382 deadline: 1731565130248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:50,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:50,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:50,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:50,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742218_1394 (size=4469) 2024-11-14T06:17:50,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742219_1395 (size=48639) 2024-11-14T06:17:50,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:50,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1731565130367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:50,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:50,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:50,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57382 deadline: 1731565130367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:50,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:50,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:50,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:50,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:50,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:50,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:50,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57382 deadline: 1731565130587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:50,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:50,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:50,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:50,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:50,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1731565130587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:50,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:50,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:50,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:50,637 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/05d91ff6c48348c8be5a959a1a28ccbf as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/05d91ff6c48348c8be5a959a1a28ccbf 2024-11-14T06:17:50,642 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b0928b43a2d9ea8c8bf8f2250ec54df/B of 8b0928b43a2d9ea8c8bf8f2250ec54df into 05d91ff6c48348c8be5a959a1a28ccbf(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:17:50,642 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b0928b43a2d9ea8c8bf8f2250ec54df: 2024-11-14T06:17:50,642 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., storeName=8b0928b43a2d9ea8c8bf8f2250ec54df/B, priority=13, startTime=1731565070194; duration=0sec 2024-11-14T06:17:50,642 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:17:50,642 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b0928b43a2d9ea8c8bf8f2250ec54df:B 2024-11-14T06:17:50,642 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:17:50,643 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36843 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:17:50,643 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): 8b0928b43a2d9ea8c8bf8f2250ec54df/C is initiating minor compaction (all files) 2024-11-14T06:17:50,643 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b0928b43a2d9ea8c8bf8f2250ec54df/C in TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:50,643 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/ff1f0d33941d41d5aa00da668029d4d0, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/2fb8fb99f2da46a58a0da12ec59f557d, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/1576fb66e1fb4fb9b167c21a7aaa58cb] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp, totalSize=36.0 K 2024-11-14T06:17:50,644 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting ff1f0d33941d41d5aa00da668029d4d0, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731565064160 2024-11-14T06:17:50,644 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 2fb8fb99f2da46a58a0da12ec59f557d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731565067503 2024-11-14T06:17:50,644 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 1576fb66e1fb4fb9b167c21a7aaa58cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1731565069325 2024-11-14T06:17:50,650 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b0928b43a2d9ea8c8bf8f2250ec54df#C#compaction#336 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:17:50,650 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/d92ef1adaa50417a8887871651cb9682 is 50, key is test_row_0/C:col10/1731565069666/Put/seqid=0 2024-11-14T06:17:50,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742220_1396 (size=12643) 2024-11-14T06:17:50,667 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b0928b43a2d9ea8c8bf8f2250ec54df#A#compaction#335 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:17:50,667 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/4600656d24c940e2a03255f83af728bd is 175, key is test_row_0/A:col10/1731565069666/Put/seqid=0 2024-11-14T06:17:50,668 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=354, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/57eca7bc2fdd4599be2dd308ce17083f 2024-11-14T06:17:50,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/9f22a5190d814aa08dc1ba28933c14f4 is 50, key is test_row_0/B:col10/1731565070164/Put/seqid=0 2024-11-14T06:17:50,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742221_1397 (size=31597) 2024-11-14T06:17:50,684 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/4600656d24c940e2a03255f83af728bd as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/4600656d24c940e2a03255f83af728bd 2024-11-14T06:17:50,688 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b0928b43a2d9ea8c8bf8f2250ec54df/A of 8b0928b43a2d9ea8c8bf8f2250ec54df into 4600656d24c940e2a03255f83af728bd(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:17:50,688 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b0928b43a2d9ea8c8bf8f2250ec54df: 2024-11-14T06:17:50,688 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., storeName=8b0928b43a2d9ea8c8bf8f2250ec54df/A, priority=13, startTime=1731565070194; duration=0sec 2024-11-14T06:17:50,688 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:50,688 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b0928b43a2d9ea8c8bf8f2250ec54df:A 2024-11-14T06:17:50,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742222_1398 (size=12301) 2024-11-14T06:17:50,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:50,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1731565130901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:50,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:50,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:50,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:50,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:50,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57382 deadline: 1731565130903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:50,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:50,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:50,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:51,064 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/d92ef1adaa50417a8887871651cb9682 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/d92ef1adaa50417a8887871651cb9682 2024-11-14T06:17:51,074 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b0928b43a2d9ea8c8bf8f2250ec54df/C of 8b0928b43a2d9ea8c8bf8f2250ec54df into d92ef1adaa50417a8887871651cb9682(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:17:51,074 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b0928b43a2d9ea8c8bf8f2250ec54df: 2024-11-14T06:17:51,074 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., storeName=8b0928b43a2d9ea8c8bf8f2250ec54df/C, priority=13, startTime=1731565070195; duration=0sec 2024-11-14T06:17:51,074 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:51,074 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b0928b43a2d9ea8c8bf8f2250ec54df:C 2024-11-14T06:17:51,094 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/9f22a5190d814aa08dc1ba28933c14f4 2024-11-14T06:17:51,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/3dc7ced41413442bba190c64c2b5a089 is 50, key is test_row_0/C:col10/1731565070164/Put/seqid=0 2024-11-14T06:17:51,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742223_1399 (size=12301) 2024-11-14T06:17:51,129 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/3dc7ced41413442bba190c64c2b5a089 2024-11-14T06:17:51,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/57eca7bc2fdd4599be2dd308ce17083f as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/57eca7bc2fdd4599be2dd308ce17083f 2024-11-14T06:17:51,142 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/57eca7bc2fdd4599be2dd308ce17083f, entries=250, sequenceid=354, filesize=47.5 K 2024-11-14T06:17:51,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/9f22a5190d814aa08dc1ba28933c14f4 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/9f22a5190d814aa08dc1ba28933c14f4 2024-11-14T06:17:51,148 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/9f22a5190d814aa08dc1ba28933c14f4, entries=150, sequenceid=354, filesize=12.0 K 2024-11-14T06:17:51,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/3dc7ced41413442bba190c64c2b5a089 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/3dc7ced41413442bba190c64c2b5a089 2024-11-14T06:17:51,166 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/3dc7ced41413442bba190c64c2b5a089, entries=150, sequenceid=354, filesize=12.0 K 2024-11-14T06:17:51,172 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~337.88 KB/345984, currentSize=60.38 KB/61830 for 8b0928b43a2d9ea8c8bf8f2250ec54df in 977ms, sequenceid=354, compaction requested=false 2024-11-14T06:17:51,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 8b0928b43a2d9ea8c8bf8f2250ec54df: 2024-11-14T06:17:51,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:51,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-14T06:17:51,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-14T06:17:51,180 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-14T06:17:51,180 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1270 sec 2024-11-14T06:17:51,182 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 3.1320 sec 2024-11-14T06:17:51,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on 8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:51,423 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8b0928b43a2d9ea8c8bf8f2250ec54df 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-14T06:17:51,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b0928b43a2d9ea8c8bf8f2250ec54df, store=A 2024-11-14T06:17:51,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:51,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b0928b43a2d9ea8c8bf8f2250ec54df, store=B 2024-11-14T06:17:51,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:51,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b0928b43a2d9ea8c8bf8f2250ec54df, store=C 2024-11-14T06:17:51,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:51,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114789d67529e1e4969bb1f8b8780019d61_8b0928b43a2d9ea8c8bf8f2250ec54df is 50, key is test_row_0/A:col10/1731565071422/Put/seqid=0 2024-11-14T06:17:51,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742224_1400 (size=14994) 2024-11-14T06:17:51,479 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:51,483 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114789d67529e1e4969bb1f8b8780019d61_8b0928b43a2d9ea8c8bf8f2250ec54df to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114789d67529e1e4969bb1f8b8780019d61_8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:51,488 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/ffc309db89e64ccd86f743bf027453f1, store: [table=TestAcidGuarantees family=A region=8b0928b43a2d9ea8c8bf8f2250ec54df] 2024-11-14T06:17:51,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/ffc309db89e64ccd86f743bf027453f1 is 175, key is test_row_0/A:col10/1731565071422/Put/seqid=0 2024-11-14T06:17:51,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742225_1401 (size=39949) 2024-11-14T06:17:51,509 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=370, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/ffc309db89e64ccd86f743bf027453f1 2024-11-14T06:17:51,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/ba33d4fe45674a568ea02cb13336c63c is 50, key is test_row_0/B:col10/1731565071422/Put/seqid=0 2024-11-14T06:17:51,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742226_1402 (size=12301) 2024-11-14T06:17:51,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:51,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57382 deadline: 1731565131581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:51,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:51,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:51,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:51,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:51,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1731565131587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:51,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:51,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:51,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:51,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:51,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57382 deadline: 1731565131705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:51,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:51,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:51,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1731565131707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:51,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:51,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:51,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:51,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:51,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:51,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:51,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57382 deadline: 1731565131927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:51,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:51,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:51,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:51,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:51,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1731565131928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:51,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:51,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:51,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:51,960 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/ba33d4fe45674a568ea02cb13336c63c 2024-11-14T06:17:51,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/5266699cf5124e80afc4206def8e3e0d is 50, key is test_row_0/C:col10/1731565071422/Put/seqid=0 2024-11-14T06:17:52,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742227_1403 (size=12301) 2024-11-14T06:17:52,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-14T06:17:52,208 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:17:52,208 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T06:17:52,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-14T06:17:52,209 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T06:17:52,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-14T06:17:52,210 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T06:17:52,210 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T06:17:52,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:52,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57382 deadline: 1731565132247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:52,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:52,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:52,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:52,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1731565132247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:52,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:52,256 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:52,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:52,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:52,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-14T06:17:52,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-14T06:17:52,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:52,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. as already flushing 2024-11-14T06:17:52,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:52,362 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:52,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:52,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:52,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:52,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57346 deadline: 1731565132387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:52,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:52,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57348 deadline: 1731565132397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:52,400 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to ef93ef3a83c5,42283,1731564965945 for 'test_row_1' in 8b0928b43a2d9ea8c8bf8f2250ec54df of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8249 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T06:17:52,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:52,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:52,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:52,400 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to ef93ef3a83c5,42283,1731564965945 for 'test_row_1' in 8b0928b43a2d9ea8c8bf8f2250ec54df of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8253 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T06:17:52,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:52,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:52,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:52,403 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/5266699cf5124e80afc4206def8e3e0d 2024-11-14T06:17:52,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/ffc309db89e64ccd86f743bf027453f1 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/ffc309db89e64ccd86f743bf027453f1 2024-11-14T06:17:52,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:52,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57358 deadline: 1731565132407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:52,415 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to ef93ef3a83c5,42283,1731564965945 for 'test_row_2' in 8b0928b43a2d9ea8c8bf8f2250ec54df of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8254 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T06:17:52,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:52,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:52,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:52,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/ffc309db89e64ccd86f743bf027453f1, entries=200, sequenceid=370, filesize=39.0 K 2024-11-14T06:17:52,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/ba33d4fe45674a568ea02cb13336c63c as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/ba33d4fe45674a568ea02cb13336c63c 2024-11-14T06:17:52,430 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/ba33d4fe45674a568ea02cb13336c63c, entries=150, sequenceid=370, filesize=12.0 K 2024-11-14T06:17:52,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/5266699cf5124e80afc4206def8e3e0d as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/5266699cf5124e80afc4206def8e3e0d 2024-11-14T06:17:52,435 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/5266699cf5124e80afc4206def8e3e0d, entries=150, sequenceid=370, filesize=12.0 K 2024-11-14T06:17:52,436 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 8b0928b43a2d9ea8c8bf8f2250ec54df in 1013ms, sequenceid=370, compaction requested=true 2024-11-14T06:17:52,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8b0928b43a2d9ea8c8bf8f2250ec54df: 2024-11-14T06:17:52,436 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:17:52,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b0928b43a2d9ea8c8bf8f2250ec54df:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T06:17:52,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:52,437 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:17:52,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b0928b43a2d9ea8c8bf8f2250ec54df:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T06:17:52,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:52,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b0928b43a2d9ea8c8bf8f2250ec54df:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T06:17:52,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:17:52,438 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 120185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:17:52,438 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): 8b0928b43a2d9ea8c8bf8f2250ec54df/A is initiating minor compaction (all files) 2024-11-14T06:17:52,438 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b0928b43a2d9ea8c8bf8f2250ec54df/A in TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:52,438 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/4600656d24c940e2a03255f83af728bd, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/57eca7bc2fdd4599be2dd308ce17083f, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/ffc309db89e64ccd86f743bf027453f1] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp, totalSize=117.4 K 2024-11-14T06:17:52,439 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:52,439 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. files: [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/4600656d24c940e2a03255f83af728bd, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/57eca7bc2fdd4599be2dd308ce17083f, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/ffc309db89e64ccd86f743bf027453f1] 2024-11-14T06:17:52,439 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37245 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:17:52,439 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): 8b0928b43a2d9ea8c8bf8f2250ec54df/B is initiating minor compaction (all files) 2024-11-14T06:17:52,439 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b0928b43a2d9ea8c8bf8f2250ec54df/B in TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:52,439 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/05d91ff6c48348c8be5a959a1a28ccbf, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/9f22a5190d814aa08dc1ba28933c14f4, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/ba33d4fe45674a568ea02cb13336c63c] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp, totalSize=36.4 K 2024-11-14T06:17:52,439 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 05d91ff6c48348c8be5a959a1a28ccbf, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1731565069325 2024-11-14T06:17:52,439 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4600656d24c940e2a03255f83af728bd, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1731565069325 2024-11-14T06:17:52,440 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f22a5190d814aa08dc1ba28933c14f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1731565070164 2024-11-14T06:17:52,440 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57eca7bc2fdd4599be2dd308ce17083f, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1731565070112 2024-11-14T06:17:52,440 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting ba33d4fe45674a568ea02cb13336c63c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731565070243 2024-11-14T06:17:52,441 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffc309db89e64ccd86f743bf027453f1, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731565070229 2024-11-14T06:17:52,448 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b0928b43a2d9ea8c8bf8f2250ec54df#B#compaction#342 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:17:52,448 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/0827ebfe636341ae94af72e7957644a8 is 50, key is test_row_0/B:col10/1731565071422/Put/seqid=0 2024-11-14T06:17:52,469 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8b0928b43a2d9ea8c8bf8f2250ec54df] 2024-11-14T06:17:52,484 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114f0620e74ac4647c48cd1181b9ea94296_8b0928b43a2d9ea8c8bf8f2250ec54df store=[table=TestAcidGuarantees family=A region=8b0928b43a2d9ea8c8bf8f2250ec54df] 2024-11-14T06:17:52,487 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114f0620e74ac4647c48cd1181b9ea94296_8b0928b43a2d9ea8c8bf8f2250ec54df, store=[table=TestAcidGuarantees family=A region=8b0928b43a2d9ea8c8bf8f2250ec54df] 2024-11-14T06:17:52,487 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114f0620e74ac4647c48cd1181b9ea94296_8b0928b43a2d9ea8c8bf8f2250ec54df because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8b0928b43a2d9ea8c8bf8f2250ec54df] 2024-11-14T06:17:52,515 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-14T06:17:52,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:52,516 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 8b0928b43a2d9ea8c8bf8f2250ec54df 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-14T06:17:52,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b0928b43a2d9ea8c8bf8f2250ec54df, store=A 2024-11-14T06:17:52,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:52,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b0928b43a2d9ea8c8bf8f2250ec54df, store=B 2024-11-14T06:17:52,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:52,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b0928b43a2d9ea8c8bf8f2250ec54df, store=C 2024-11-14T06:17:52,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:52,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742228_1404 (size=12745) 2024-11-14T06:17:52,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-14T06:17:52,530 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/0827ebfe636341ae94af72e7957644a8 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/0827ebfe636341ae94af72e7957644a8 2024-11-14T06:17:52,535 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b0928b43a2d9ea8c8bf8f2250ec54df/B of 8b0928b43a2d9ea8c8bf8f2250ec54df into 0827ebfe636341ae94af72e7957644a8(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:17:52,535 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b0928b43a2d9ea8c8bf8f2250ec54df: 2024-11-14T06:17:52,535 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., storeName=8b0928b43a2d9ea8c8bf8f2250ec54df/B, priority=13, startTime=1731565072436; duration=0sec 2024-11-14T06:17:52,535 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:17:52,535 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b0928b43a2d9ea8c8bf8f2250ec54df:B 2024-11-14T06:17:52,535 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:17:52,536 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37245 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:17:52,537 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): 8b0928b43a2d9ea8c8bf8f2250ec54df/C is initiating minor compaction (all files) 2024-11-14T06:17:52,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742229_1405 (size=4469) 2024-11-14T06:17:52,537 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b0928b43a2d9ea8c8bf8f2250ec54df/C in TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:52,537 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/d92ef1adaa50417a8887871651cb9682, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/3dc7ced41413442bba190c64c2b5a089, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/5266699cf5124e80afc4206def8e3e0d] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp, totalSize=36.4 K 2024-11-14T06:17:52,537 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting d92ef1adaa50417a8887871651cb9682, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1731565069325 2024-11-14T06:17:52,538 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 3dc7ced41413442bba190c64c2b5a089, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1731565070164 2024-11-14T06:17:52,545 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b0928b43a2d9ea8c8bf8f2250ec54df#A#compaction#343 average throughput is 0.32 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:17:52,546 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/ba4c2abe2e54438f91d0f9f743a69a09 is 175, key is test_row_0/A:col10/1731565071422/Put/seqid=0 2024-11-14T06:17:52,546 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 5266699cf5124e80afc4206def8e3e0d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731565070243 2024-11-14T06:17:52,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114d23c922396674932aa3e9e556fac2262_8b0928b43a2d9ea8c8bf8f2250ec54df is 50, key is test_row_0/A:col10/1731565071581/Put/seqid=0 2024-11-14T06:17:52,603 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b0928b43a2d9ea8c8bf8f2250ec54df#C#compaction#345 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:17:52,603 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/313bf7bdef2745928d0c51106cd67b1d is 50, key is test_row_0/C:col10/1731565071422/Put/seqid=0 2024-11-14T06:17:52,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742230_1406 (size=31699) 2024-11-14T06:17:52,644 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/ba4c2abe2e54438f91d0f9f743a69a09 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/ba4c2abe2e54438f91d0f9f743a69a09 2024-11-14T06:17:52,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742232_1408 (size=12745) 2024-11-14T06:17:52,668 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/313bf7bdef2745928d0c51106cd67b1d as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/313bf7bdef2745928d0c51106cd67b1d 2024-11-14T06:17:52,672 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b0928b43a2d9ea8c8bf8f2250ec54df/A of 8b0928b43a2d9ea8c8bf8f2250ec54df into ba4c2abe2e54438f91d0f9f743a69a09(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:17:52,672 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b0928b43a2d9ea8c8bf8f2250ec54df: 2024-11-14T06:17:52,672 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., storeName=8b0928b43a2d9ea8c8bf8f2250ec54df/A, priority=13, startTime=1731565072436; duration=0sec 2024-11-14T06:17:52,672 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:52,672 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b0928b43a2d9ea8c8bf8f2250ec54df:A 2024-11-14T06:17:52,673 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b0928b43a2d9ea8c8bf8f2250ec54df/C of 8b0928b43a2d9ea8c8bf8f2250ec54df into 313bf7bdef2745928d0c51106cd67b1d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:17:52,674 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b0928b43a2d9ea8c8bf8f2250ec54df: 2024-11-14T06:17:52,674 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., storeName=8b0928b43a2d9ea8c8bf8f2250ec54df/C, priority=13, startTime=1731565072437; duration=0sec 2024-11-14T06:17:52,674 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:52,674 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b0928b43a2d9ea8c8bf8f2250ec54df:C 2024-11-14T06:17:52,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742231_1407 (size=12454) 2024-11-14T06:17:52,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on 8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:52,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. as already flushing 2024-11-14T06:17:52,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:52,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1731565132816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:52,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:52,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57382 deadline: 1731565132817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:52,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:52,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:52,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:52,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:52,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:52,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:52,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-14T06:17:52,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:52,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1731565132937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:52,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:52,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:52,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57382 deadline: 1731565132937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:52,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:52,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:52,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:52,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:52,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:53,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:53,107 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114d23c922396674932aa3e9e556fac2262_8b0928b43a2d9ea8c8bf8f2250ec54df to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d23c922396674932aa3e9e556fac2262_8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:53,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/1e38abdd38e249f08cb3f4f2b298ddd8, store: [table=TestAcidGuarantees family=A region=8b0928b43a2d9ea8c8bf8f2250ec54df] 2024-11-14T06:17:53,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/1e38abdd38e249f08cb3f4f2b298ddd8 is 175, key is test_row_0/A:col10/1731565071581/Put/seqid=0 2024-11-14T06:17:53,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742233_1409 (size=31255) 2024-11-14T06:17:53,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:53,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57382 deadline: 1731565133148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:53,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:53,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:53,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:53,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:53,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1731565133157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:53,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:53,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:53,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:53,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-14T06:17:53,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:53,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57382 deadline: 1731565133468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:53,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:53,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:53,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:53,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:53,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1731565133477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:53,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:53,497 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:53,497 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:53,550 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=389, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/1e38abdd38e249f08cb3f4f2b298ddd8 2024-11-14T06:17:53,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/f7f5754e0eb1423fadefd9015041b0d9 is 50, key is test_row_0/B:col10/1731565071581/Put/seqid=0 2024-11-14T06:17:53,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742234_1410 (size=12301) 2024-11-14T06:17:53,583 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/f7f5754e0eb1423fadefd9015041b0d9 2024-11-14T06:17:53,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/e9e1048230e94d1db057b0be79fb80dd is 50, key is test_row_0/C:col10/1731565071581/Put/seqid=0 2024-11-14T06:17:53,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742235_1411 (size=12301) 2024-11-14T06:17:53,641 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/e9e1048230e94d1db057b0be79fb80dd 2024-11-14T06:17:53,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/1e38abdd38e249f08cb3f4f2b298ddd8 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/1e38abdd38e249f08cb3f4f2b298ddd8 2024-11-14T06:17:53,660 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/1e38abdd38e249f08cb3f4f2b298ddd8, entries=150, sequenceid=389, filesize=30.5 K 2024-11-14T06:17:53,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/f7f5754e0eb1423fadefd9015041b0d9 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/f7f5754e0eb1423fadefd9015041b0d9 2024-11-14T06:17:53,664 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/f7f5754e0eb1423fadefd9015041b0d9, entries=150, sequenceid=389, filesize=12.0 K 2024-11-14T06:17:53,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/e9e1048230e94d1db057b0be79fb80dd as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/e9e1048230e94d1db057b0be79fb80dd 2024-11-14T06:17:53,669 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/e9e1048230e94d1db057b0be79fb80dd, entries=150, sequenceid=389, filesize=12.0 K 2024-11-14T06:17:53,673 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 8b0928b43a2d9ea8c8bf8f2250ec54df in 1157ms, sequenceid=389, compaction requested=false 2024-11-14T06:17:53,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 8b0928b43a2d9ea8c8bf8f2250ec54df: 2024-11-14T06:17:53,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:53,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-14T06:17:53,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-14T06:17:53,682 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-14T06:17:53,682 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4680 sec 2024-11-14T06:17:53,684 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.4750 sec 2024-11-14T06:17:53,994 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8b0928b43a2d9ea8c8bf8f2250ec54df 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-14T06:17:53,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b0928b43a2d9ea8c8bf8f2250ec54df, store=A 2024-11-14T06:17:53,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:53,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b0928b43a2d9ea8c8bf8f2250ec54df, store=B 2024-11-14T06:17:53,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:53,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b0928b43a2d9ea8c8bf8f2250ec54df, store=C 2024-11-14T06:17:53,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:53,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on 8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:54,015 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114e74e176aa20b4674ad0e17f4bf74a13c_8b0928b43a2d9ea8c8bf8f2250ec54df is 50, key is test_row_0/A:col10/1731565073992/Put/seqid=0 2024-11-14T06:17:54,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742236_1412 (size=14994) 2024-11-14T06:17:54,027 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:54,032 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114e74e176aa20b4674ad0e17f4bf74a13c_8b0928b43a2d9ea8c8bf8f2250ec54df to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e74e176aa20b4674ad0e17f4bf74a13c_8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:54,033 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/f4f1dd27e1d64ab18ce6c97b3b8d550c, store: [table=TestAcidGuarantees family=A region=8b0928b43a2d9ea8c8bf8f2250ec54df] 2024-11-14T06:17:54,035 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/f4f1dd27e1d64ab18ce6c97b3b8d550c is 175, key is test_row_0/A:col10/1731565073992/Put/seqid=0 2024-11-14T06:17:54,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742237_1413 (size=39949) 2024-11-14T06:17:54,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:54,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1731565134101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:54,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:54,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:54,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57382 deadline: 1731565134101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:54,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:54,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:54,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:54,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:54,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:54,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:54,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1731565134227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:54,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:54,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57382 deadline: 1731565134227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:54,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:54,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:54,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:54,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:54,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:54,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:54,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-14T06:17:54,358 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:17:54,359 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T06:17:54,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-14T06:17:54,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-14T06:17:54,361 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T06:17:54,361 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T06:17:54,362 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T06:17:54,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:54,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1731565134447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:54,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:54,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:54,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:54,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:54,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57382 deadline: 1731565134447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:54,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:54,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:54,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:54,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-14T06:17:54,481 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=406, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/f4f1dd27e1d64ab18ce6c97b3b8d550c 2024-11-14T06:17:54,514 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-14T06:17:54,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:54,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. as already flushing 2024-11-14T06:17:54,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:54,515 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:54,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:54,516 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/b074ebc37ced4705be9c8aa9d27e2aa2 is 50, key is test_row_0/B:col10/1731565073992/Put/seqid=0 2024-11-14T06:17:54,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:54,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742238_1414 (size=12301) 2024-11-14T06:17:54,520 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/b074ebc37ced4705be9c8aa9d27e2aa2 2024-11-14T06:17:54,527 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/9d984f94cf274ecdb4303d3a752ef71d is 50, key is test_row_0/C:col10/1731565073992/Put/seqid=0 2024-11-14T06:17:54,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742239_1415 (size=12301) 2024-11-14T06:17:54,542 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/9d984f94cf274ecdb4303d3a752ef71d 2024-11-14T06:17:54,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/f4f1dd27e1d64ab18ce6c97b3b8d550c as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/f4f1dd27e1d64ab18ce6c97b3b8d550c 2024-11-14T06:17:54,559 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/f4f1dd27e1d64ab18ce6c97b3b8d550c, entries=200, sequenceid=406, filesize=39.0 K 2024-11-14T06:17:54,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/b074ebc37ced4705be9c8aa9d27e2aa2 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/b074ebc37ced4705be9c8aa9d27e2aa2 2024-11-14T06:17:54,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/b074ebc37ced4705be9c8aa9d27e2aa2, entries=150, sequenceid=406, filesize=12.0 K 2024-11-14T06:17:54,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/9d984f94cf274ecdb4303d3a752ef71d as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/9d984f94cf274ecdb4303d3a752ef71d 2024-11-14T06:17:54,571 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/9d984f94cf274ecdb4303d3a752ef71d, entries=150, sequenceid=406, filesize=12.0 K 2024-11-14T06:17:54,572 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 8b0928b43a2d9ea8c8bf8f2250ec54df in 578ms, sequenceid=406, compaction requested=true 2024-11-14T06:17:54,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8b0928b43a2d9ea8c8bf8f2250ec54df: 2024-11-14T06:17:54,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b0928b43a2d9ea8c8bf8f2250ec54df:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T06:17:54,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:54,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b0928b43a2d9ea8c8bf8f2250ec54df:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T06:17:54,572 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:17:54,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:54,572 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:17:54,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b0928b43a2d9ea8c8bf8f2250ec54df:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T06:17:54,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:17:54,573 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:17:54,573 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102903 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:17:54,573 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): 8b0928b43a2d9ea8c8bf8f2250ec54df/B is initiating minor compaction (all files) 2024-11-14T06:17:54,573 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): 8b0928b43a2d9ea8c8bf8f2250ec54df/A is initiating minor compaction (all files) 2024-11-14T06:17:54,573 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b0928b43a2d9ea8c8bf8f2250ec54df/B in TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:54,573 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b0928b43a2d9ea8c8bf8f2250ec54df/A in TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:54,573 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/0827ebfe636341ae94af72e7957644a8, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/f7f5754e0eb1423fadefd9015041b0d9, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/b074ebc37ced4705be9c8aa9d27e2aa2] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp, totalSize=36.5 K 2024-11-14T06:17:54,573 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/ba4c2abe2e54438f91d0f9f743a69a09, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/1e38abdd38e249f08cb3f4f2b298ddd8, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/f4f1dd27e1d64ab18ce6c97b3b8d550c] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp, totalSize=100.5 K 2024-11-14T06:17:54,573 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:54,573 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. files: [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/ba4c2abe2e54438f91d0f9f743a69a09, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/1e38abdd38e249f08cb3f4f2b298ddd8, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/f4f1dd27e1d64ab18ce6c97b3b8d550c] 2024-11-14T06:17:54,574 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 0827ebfe636341ae94af72e7957644a8, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731565070243 2024-11-14T06:17:54,576 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting f7f5754e0eb1423fadefd9015041b0d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1731565071560 2024-11-14T06:17:54,576 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba4c2abe2e54438f91d0f9f743a69a09, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731565070243 2024-11-14T06:17:54,576 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e38abdd38e249f08cb3f4f2b298ddd8, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1731565071560 2024-11-14T06:17:54,576 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting b074ebc37ced4705be9c8aa9d27e2aa2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1731565072780 2024-11-14T06:17:54,577 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4f1dd27e1d64ab18ce6c97b3b8d550c, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1731565072780 2024-11-14T06:17:54,584 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8b0928b43a2d9ea8c8bf8f2250ec54df] 2024-11-14T06:17:54,585 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b0928b43a2d9ea8c8bf8f2250ec54df#B#compaction#351 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:17:54,585 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/459a4d5233104415b82fad0469d32e9c is 50, key is test_row_0/B:col10/1731565073992/Put/seqid=0 2024-11-14T06:17:54,589 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114526c428691ea4bf0a7c0c09f9af778c6_8b0928b43a2d9ea8c8bf8f2250ec54df store=[table=TestAcidGuarantees family=A region=8b0928b43a2d9ea8c8bf8f2250ec54df] 2024-11-14T06:17:54,591 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114526c428691ea4bf0a7c0c09f9af778c6_8b0928b43a2d9ea8c8bf8f2250ec54df, store=[table=TestAcidGuarantees family=A region=8b0928b43a2d9ea8c8bf8f2250ec54df] 2024-11-14T06:17:54,591 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114526c428691ea4bf0a7c0c09f9af778c6_8b0928b43a2d9ea8c8bf8f2250ec54df because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8b0928b43a2d9ea8c8bf8f2250ec54df] 2024-11-14T06:17:54,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742240_1416 (size=12847) 2024-11-14T06:17:54,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742241_1417 (size=4469) 2024-11-14T06:17:54,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-14T06:17:54,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:54,668 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 8b0928b43a2d9ea8c8bf8f2250ec54df 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-14T06:17:54,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b0928b43a2d9ea8c8bf8f2250ec54df, store=A 2024-11-14T06:17:54,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:54,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b0928b43a2d9ea8c8bf8f2250ec54df, store=B 2024-11-14T06:17:54,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:54,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b0928b43a2d9ea8c8bf8f2250ec54df, store=C 2024-11-14T06:17:54,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:54,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411148745ac0a46cb4d78b6b74dc5ef14ca42_8b0928b43a2d9ea8c8bf8f2250ec54df is 50, key is test_row_0/A:col10/1731565074031/Put/seqid=0 2024-11-14T06:17:54,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-14T06:17:54,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742242_1418 (size=12454) 2024-11-14T06:17:54,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:54,690 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411148745ac0a46cb4d78b6b74dc5ef14ca42_8b0928b43a2d9ea8c8bf8f2250ec54df to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411148745ac0a46cb4d78b6b74dc5ef14ca42_8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:54,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/91f66ef144a0422cb80a24413449c73d, store: [table=TestAcidGuarantees family=A region=8b0928b43a2d9ea8c8bf8f2250ec54df] 2024-11-14T06:17:54,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/91f66ef144a0422cb80a24413449c73d is 175, key is test_row_0/A:col10/1731565074031/Put/seqid=0 2024-11-14T06:17:54,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742243_1419 (size=31255) 2024-11-14T06:17:54,695 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=424, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/91f66ef144a0422cb80a24413449c73d 2024-11-14T06:17:54,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/1ed853bc229c48d88d270aaed7f303cf is 50, key is test_row_0/B:col10/1731565074031/Put/seqid=0 2024-11-14T06:17:54,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742244_1420 (size=12301) 2024-11-14T06:17:54,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. as already flushing 2024-11-14T06:17:54,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on 8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:54,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:54,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57382 deadline: 1731565134829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:54,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:54,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:54,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:54,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:54,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1731565134833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:54,841 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:54,841 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:54,841 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:54,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:54,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57382 deadline: 1731565134947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:54,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:54,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1731565134948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:54,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:54,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:54,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:54,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:54,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:54,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:54,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-14T06:17:55,010 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b0928b43a2d9ea8c8bf8f2250ec54df#A#compaction#352 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:17:55,011 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/aa9ea5ddfd684477a5918cdd0f2d7e7c is 175, key is test_row_0/A:col10/1731565073992/Put/seqid=0 2024-11-14T06:17:55,012 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/459a4d5233104415b82fad0469d32e9c as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/459a4d5233104415b82fad0469d32e9c 2024-11-14T06:17:55,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742245_1421 (size=31801) 2024-11-14T06:17:55,018 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b0928b43a2d9ea8c8bf8f2250ec54df/B of 8b0928b43a2d9ea8c8bf8f2250ec54df into 459a4d5233104415b82fad0469d32e9c(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:17:55,018 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b0928b43a2d9ea8c8bf8f2250ec54df: 2024-11-14T06:17:55,018 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., storeName=8b0928b43a2d9ea8c8bf8f2250ec54df/B, priority=13, startTime=1731565074572; duration=0sec 2024-11-14T06:17:55,018 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:17:55,018 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b0928b43a2d9ea8c8bf8f2250ec54df:B 2024-11-14T06:17:55,018 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:17:55,019 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:17:55,019 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): 8b0928b43a2d9ea8c8bf8f2250ec54df/C is initiating minor compaction (all files) 2024-11-14T06:17:55,019 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b0928b43a2d9ea8c8bf8f2250ec54df/C in TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:55,019 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/313bf7bdef2745928d0c51106cd67b1d, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/e9e1048230e94d1db057b0be79fb80dd, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/9d984f94cf274ecdb4303d3a752ef71d] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp, totalSize=36.5 K 2024-11-14T06:17:55,020 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 313bf7bdef2745928d0c51106cd67b1d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731565070243 2024-11-14T06:17:55,020 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting e9e1048230e94d1db057b0be79fb80dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1731565071560 2024-11-14T06:17:55,021 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d984f94cf274ecdb4303d3a752ef71d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1731565072780 2024-11-14T06:17:55,023 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/aa9ea5ddfd684477a5918cdd0f2d7e7c as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/aa9ea5ddfd684477a5918cdd0f2d7e7c 2024-11-14T06:17:55,030 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b0928b43a2d9ea8c8bf8f2250ec54df#C#compaction#355 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:17:55,030 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/727036d956664ae3b4b14334c0122ddc is 50, key is test_row_0/C:col10/1731565073992/Put/seqid=0 2024-11-14T06:17:55,037 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b0928b43a2d9ea8c8bf8f2250ec54df/A of 8b0928b43a2d9ea8c8bf8f2250ec54df into aa9ea5ddfd684477a5918cdd0f2d7e7c(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:17:55,037 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b0928b43a2d9ea8c8bf8f2250ec54df: 2024-11-14T06:17:55,037 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., storeName=8b0928b43a2d9ea8c8bf8f2250ec54df/A, priority=13, startTime=1731565074572; duration=0sec 2024-11-14T06:17:55,037 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:55,037 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b0928b43a2d9ea8c8bf8f2250ec54df:A 2024-11-14T06:17:55,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742246_1422 (size=12847) 2024-11-14T06:17:55,106 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/1ed853bc229c48d88d270aaed7f303cf 2024-11-14T06:17:55,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/972b9c490a764fc5ae7421dc9b7c8197 is 50, key is test_row_0/C:col10/1731565074031/Put/seqid=0 2024-11-14T06:17:55,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742247_1423 (size=12301) 2024-11-14T06:17:55,133 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/972b9c490a764fc5ae7421dc9b7c8197 2024-11-14T06:17:55,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/91f66ef144a0422cb80a24413449c73d as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/91f66ef144a0422cb80a24413449c73d 2024-11-14T06:17:55,151 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/91f66ef144a0422cb80a24413449c73d, entries=150, sequenceid=424, filesize=30.5 K 2024-11-14T06:17:55,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/1ed853bc229c48d88d270aaed7f303cf as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/1ed853bc229c48d88d270aaed7f303cf 2024-11-14T06:17:55,156 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/1ed853bc229c48d88d270aaed7f303cf, entries=150, sequenceid=424, filesize=12.0 K 2024-11-14T06:17:55,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/972b9c490a764fc5ae7421dc9b7c8197 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/972b9c490a764fc5ae7421dc9b7c8197 2024-11-14T06:17:55,164 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/972b9c490a764fc5ae7421dc9b7c8197, entries=150, sequenceid=424, filesize=12.0 K 2024-11-14T06:17:55,165 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 8b0928b43a2d9ea8c8bf8f2250ec54df in 497ms, sequenceid=424, compaction requested=false 2024-11-14T06:17:55,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 8b0928b43a2d9ea8c8bf8f2250ec54df: 2024-11-14T06:17:55,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:55,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-14T06:17:55,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-14T06:17:55,169 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-14T06:17:55,169 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 806 msec 2024-11-14T06:17:55,172 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 811 msec 2024-11-14T06:17:55,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on 8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:55,185 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8b0928b43a2d9ea8c8bf8f2250ec54df 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-14T06:17:55,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b0928b43a2d9ea8c8bf8f2250ec54df, store=A 2024-11-14T06:17:55,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:55,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b0928b43a2d9ea8c8bf8f2250ec54df, store=B 2024-11-14T06:17:55,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:55,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b0928b43a2d9ea8c8bf8f2250ec54df, store=C 2024-11-14T06:17:55,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:55,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114d3bda702e063492e93caf9c393f3b513_8b0928b43a2d9ea8c8bf8f2250ec54df is 50, key is test_row_0/A:col10/1731565075181/Put/seqid=0 2024-11-14T06:17:55,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742248_1424 (size=17534) 2024-11-14T06:17:55,229 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:55,233 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114d3bda702e063492e93caf9c393f3b513_8b0928b43a2d9ea8c8bf8f2250ec54df to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d3bda702e063492e93caf9c393f3b513_8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:55,234 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/32bdf305fa6743a4bd9b9c48183ac2c2, store: [table=TestAcidGuarantees family=A region=8b0928b43a2d9ea8c8bf8f2250ec54df] 2024-11-14T06:17:55,234 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/32bdf305fa6743a4bd9b9c48183ac2c2 is 175, key is test_row_0/A:col10/1731565075181/Put/seqid=0 2024-11-14T06:17:55,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742249_1425 (size=48639) 2024-11-14T06:17:55,254 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=442, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/32bdf305fa6743a4bd9b9c48183ac2c2 2024-11-14T06:17:55,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/11137ea950ae4b62a1f9cfe195cb9d3b is 50, key is test_row_0/B:col10/1731565075181/Put/seqid=0 2024-11-14T06:17:55,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742250_1426 (size=12301) 2024-11-14T06:17:55,298 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/11137ea950ae4b62a1f9cfe195cb9d3b 2024-11-14T06:17:55,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:55,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1731565135288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:55,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:55,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:55,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:55,309 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/4c7ca30a91164711bb23350304e79c5d is 50, key is test_row_0/C:col10/1731565075181/Put/seqid=0 2024-11-14T06:17:55,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:55,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57382 deadline: 1731565135289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:55,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:55,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:55,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:55,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742251_1427 (size=12301) 2024-11-14T06:17:55,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:55,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1731565135417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:55,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:55,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:55,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:55,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:55,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57382 deadline: 1731565135417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:55,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:55,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:55,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:55,445 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/727036d956664ae3b4b14334c0122ddc as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/727036d956664ae3b4b14334c0122ddc 2024-11-14T06:17:55,451 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b0928b43a2d9ea8c8bf8f2250ec54df/C of 8b0928b43a2d9ea8c8bf8f2250ec54df into 727036d956664ae3b4b14334c0122ddc(size=12.5 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:17:55,451 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b0928b43a2d9ea8c8bf8f2250ec54df: 2024-11-14T06:17:55,451 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., storeName=8b0928b43a2d9ea8c8bf8f2250ec54df/C, priority=13, startTime=1731565074572; duration=0sec 2024-11-14T06:17:55,451 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:55,451 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b0928b43a2d9ea8c8bf8f2250ec54df:C 2024-11-14T06:17:55,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-14T06:17:55,498 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:17:55,498 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T06:17:55,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-14T06:17:55,500 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T06:17:55,502 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T06:17:55,502 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T06:17:55,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-14T06:17:55,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-14T06:17:55,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:55,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1731565135637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:55,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:55,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57382 deadline: 1731565135637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:55,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:55,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:55,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:55,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:55,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:55,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:55,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-14T06:17:55,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:55,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. as already flushing 2024-11-14T06:17:55,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:55,656 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:55,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:55,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:17:55,723 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/4c7ca30a91164711bb23350304e79c5d 2024-11-14T06:17:55,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/32bdf305fa6743a4bd9b9c48183ac2c2 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/32bdf305fa6743a4bd9b9c48183ac2c2 2024-11-14T06:17:55,734 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/32bdf305fa6743a4bd9b9c48183ac2c2, entries=250, sequenceid=442, filesize=47.5 K 2024-11-14T06:17:55,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/11137ea950ae4b62a1f9cfe195cb9d3b as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/11137ea950ae4b62a1f9cfe195cb9d3b 2024-11-14T06:17:55,744 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/11137ea950ae4b62a1f9cfe195cb9d3b, entries=150, sequenceid=442, filesize=12.0 K 2024-11-14T06:17:55,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/4c7ca30a91164711bb23350304e79c5d as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/4c7ca30a91164711bb23350304e79c5d 2024-11-14T06:17:55,752 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/4c7ca30a91164711bb23350304e79c5d, entries=150, sequenceid=442, filesize=12.0 K 2024-11-14T06:17:55,754 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 8b0928b43a2d9ea8c8bf8f2250ec54df in 568ms, sequenceid=442, compaction requested=true 2024-11-14T06:17:55,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8b0928b43a2d9ea8c8bf8f2250ec54df: 2024-11-14T06:17:55,754 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:17:55,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b0928b43a2d9ea8c8bf8f2250ec54df:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T06:17:55,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:55,754 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:17:55,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b0928b43a2d9ea8c8bf8f2250ec54df:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T06:17:55,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:55,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b0928b43a2d9ea8c8bf8f2250ec54df:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T06:17:55,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:17:55,755 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111695 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:17:55,755 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): 8b0928b43a2d9ea8c8bf8f2250ec54df/A is initiating minor compaction (all files) 2024-11-14T06:17:55,755 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b0928b43a2d9ea8c8bf8f2250ec54df/A in TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:55,755 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/aa9ea5ddfd684477a5918cdd0f2d7e7c, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/91f66ef144a0422cb80a24413449c73d, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/32bdf305fa6743a4bd9b9c48183ac2c2] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp, totalSize=109.1 K 2024-11-14T06:17:55,755 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:55,755 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. files: [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/aa9ea5ddfd684477a5918cdd0f2d7e7c, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/91f66ef144a0422cb80a24413449c73d, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/32bdf305fa6743a4bd9b9c48183ac2c2] 2024-11-14T06:17:55,755 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:17:55,755 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa9ea5ddfd684477a5918cdd0f2d7e7c, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1731565072780 2024-11-14T06:17:55,755 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): 8b0928b43a2d9ea8c8bf8f2250ec54df/B is initiating minor compaction (all files) 2024-11-14T06:17:55,756 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b0928b43a2d9ea8c8bf8f2250ec54df/B in TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:55,756 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/459a4d5233104415b82fad0469d32e9c, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/1ed853bc229c48d88d270aaed7f303cf, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/11137ea950ae4b62a1f9cfe195cb9d3b] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp, totalSize=36.6 K 2024-11-14T06:17:55,756 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 459a4d5233104415b82fad0469d32e9c, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1731565072780 2024-11-14T06:17:55,756 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91f66ef144a0422cb80a24413449c73d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1731565074031 2024-11-14T06:17:55,756 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ed853bc229c48d88d270aaed7f303cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1731565074031 2024-11-14T06:17:55,756 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32bdf305fa6743a4bd9b9c48183ac2c2, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1731565074819 2024-11-14T06:17:55,756 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 11137ea950ae4b62a1f9cfe195cb9d3b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1731565074832 2024-11-14T06:17:55,772 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8b0928b43a2d9ea8c8bf8f2250ec54df] 2024-11-14T06:17:55,782 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b0928b43a2d9ea8c8bf8f2250ec54df#B#compaction#361 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:17:55,783 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/c34ad7694753407d8251d66ed9f322af is 50, key is test_row_0/B:col10/1731565075181/Put/seqid=0 2024-11-14T06:17:55,783 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114b5df9aa0b1fe4322a09607cd9ecffeb5_8b0928b43a2d9ea8c8bf8f2250ec54df store=[table=TestAcidGuarantees family=A region=8b0928b43a2d9ea8c8bf8f2250ec54df] 2024-11-14T06:17:55,785 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114b5df9aa0b1fe4322a09607cd9ecffeb5_8b0928b43a2d9ea8c8bf8f2250ec54df, store=[table=TestAcidGuarantees family=A region=8b0928b43a2d9ea8c8bf8f2250ec54df] 2024-11-14T06:17:55,785 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b5df9aa0b1fe4322a09607cd9ecffeb5_8b0928b43a2d9ea8c8bf8f2250ec54df because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8b0928b43a2d9ea8c8bf8f2250ec54df] 2024-11-14T06:17:55,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742252_1428 (size=12949) 2024-11-14T06:17:55,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742253_1429 (size=4469) 2024-11-14T06:17:55,802 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b0928b43a2d9ea8c8bf8f2250ec54df#A#compaction#360 average throughput is 0.81 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:17:55,803 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/c09579a40926430397bbb3d291f9ac2c is 175, key is test_row_0/A:col10/1731565075181/Put/seqid=0 2024-11-14T06:17:55,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-14T06:17:55,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:55,809 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 8b0928b43a2d9ea8c8bf8f2250ec54df 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-14T06:17:55,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b0928b43a2d9ea8c8bf8f2250ec54df, store=A 2024-11-14T06:17:55,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:55,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b0928b43a2d9ea8c8bf8f2250ec54df, store=B 2024-11-14T06:17:55,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:55,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b0928b43a2d9ea8c8bf8f2250ec54df, store=C 2024-11-14T06:17:55,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:55,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742254_1430 (size=31903) 2024-11-14T06:17:55,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-14T06:17:55,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114af30a4b97b5747ee970456e94f7b3c9c_8b0928b43a2d9ea8c8bf8f2250ec54df is 50, key is test_row_0/A:col10/1731565075259/Put/seqid=0 2024-11-14T06:17:55,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742255_1431 (size=12454) 2024-11-14T06:17:55,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:55,837 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114af30a4b97b5747ee970456e94f7b3c9c_8b0928b43a2d9ea8c8bf8f2250ec54df to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114af30a4b97b5747ee970456e94f7b3c9c_8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:55,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/f70439a41cfb4b01829bea4c190ab0d8, store: [table=TestAcidGuarantees family=A region=8b0928b43a2d9ea8c8bf8f2250ec54df] 2024-11-14T06:17:55,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/f70439a41cfb4b01829bea4c190ab0d8 is 175, key is test_row_0/A:col10/1731565075259/Put/seqid=0 2024-11-14T06:17:55,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742256_1432 (size=31255) 2024-11-14T06:17:55,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on 8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:55,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. as already flushing 2024-11-14T06:17:56,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:56,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57382 deadline: 1731565136023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:56,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:56,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1731565136028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:56,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:56,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:56,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:56,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:56,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:56,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:56,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-14T06:17:56,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:56,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57382 deadline: 1731565136148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:56,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:56,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:56,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:56,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:56,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1731565136148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:56,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:56,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:56,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:56,212 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/c34ad7694753407d8251d66ed9f322af as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/c34ad7694753407d8251d66ed9f322af 2024-11-14T06:17:56,219 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b0928b43a2d9ea8c8bf8f2250ec54df/B of 8b0928b43a2d9ea8c8bf8f2250ec54df into c34ad7694753407d8251d66ed9f322af(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:17:56,219 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b0928b43a2d9ea8c8bf8f2250ec54df: 2024-11-14T06:17:56,219 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., storeName=8b0928b43a2d9ea8c8bf8f2250ec54df/B, priority=13, startTime=1731565075754; duration=0sec 2024-11-14T06:17:56,219 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:17:56,219 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b0928b43a2d9ea8c8bf8f2250ec54df:B 2024-11-14T06:17:56,219 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:17:56,220 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/c09579a40926430397bbb3d291f9ac2c as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/c09579a40926430397bbb3d291f9ac2c 2024-11-14T06:17:56,221 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:17:56,221 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): 8b0928b43a2d9ea8c8bf8f2250ec54df/C is initiating minor compaction (all files) 2024-11-14T06:17:56,221 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b0928b43a2d9ea8c8bf8f2250ec54df/C in TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:56,221 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/727036d956664ae3b4b14334c0122ddc, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/972b9c490a764fc5ae7421dc9b7c8197, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/4c7ca30a91164711bb23350304e79c5d] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp, totalSize=36.6 K 2024-11-14T06:17:56,222 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 727036d956664ae3b4b14334c0122ddc, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1731565072780 2024-11-14T06:17:56,222 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 972b9c490a764fc5ae7421dc9b7c8197, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1731565074031 2024-11-14T06:17:56,222 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c7ca30a91164711bb23350304e79c5d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1731565074832 2024-11-14T06:17:56,225 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b0928b43a2d9ea8c8bf8f2250ec54df/A of 8b0928b43a2d9ea8c8bf8f2250ec54df into c09579a40926430397bbb3d291f9ac2c(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:17:56,225 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b0928b43a2d9ea8c8bf8f2250ec54df: 2024-11-14T06:17:56,225 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., storeName=8b0928b43a2d9ea8c8bf8f2250ec54df/A, priority=13, startTime=1731565075754; duration=0sec 2024-11-14T06:17:56,225 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:56,225 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b0928b43a2d9ea8c8bf8f2250ec54df:A 2024-11-14T06:17:56,232 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b0928b43a2d9ea8c8bf8f2250ec54df#C#compaction#363 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:17:56,232 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/e2858f6ac6e24bafa8bae6ca99889ea5 is 50, key is test_row_0/C:col10/1731565075181/Put/seqid=0 2024-11-14T06:17:56,245 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=459, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/f70439a41cfb4b01829bea4c190ab0d8 2024-11-14T06:17:56,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742257_1433 (size=12949) 2024-11-14T06:17:56,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/16700f0f53ef4154b9f5a6eecb67a163 is 50, key is test_row_0/B:col10/1731565075259/Put/seqid=0 2024-11-14T06:17:56,271 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/e2858f6ac6e24bafa8bae6ca99889ea5 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/e2858f6ac6e24bafa8bae6ca99889ea5 2024-11-14T06:17:56,277 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b0928b43a2d9ea8c8bf8f2250ec54df/C of 8b0928b43a2d9ea8c8bf8f2250ec54df into e2858f6ac6e24bafa8bae6ca99889ea5(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:17:56,277 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b0928b43a2d9ea8c8bf8f2250ec54df: 2024-11-14T06:17:56,277 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., storeName=8b0928b43a2d9ea8c8bf8f2250ec54df/C, priority=13, startTime=1731565075754; duration=0sec 2024-11-14T06:17:56,277 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:56,277 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b0928b43a2d9ea8c8bf8f2250ec54df:C 2024-11-14T06:17:56,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742258_1434 (size=12301) 2024-11-14T06:17:56,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:56,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57382 deadline: 1731565136370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:56,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:56,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:56,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:56,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:56,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1731565136370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:56,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:56,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:56,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:56,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-14T06:17:56,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:56,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57382 deadline: 1731565136677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:56,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:56,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:56,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:56,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:56,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1731565136687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:56,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:56,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:56,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:56,712 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/16700f0f53ef4154b9f5a6eecb67a163 2024-11-14T06:17:56,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/6fa2c3a1aeb84813a336628daebfb6b2 is 50, key is test_row_0/C:col10/1731565075259/Put/seqid=0 2024-11-14T06:17:56,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742259_1435 (size=12301) 2024-11-14T06:17:56,730 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/6fa2c3a1aeb84813a336628daebfb6b2 2024-11-14T06:17:56,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/f70439a41cfb4b01829bea4c190ab0d8 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/f70439a41cfb4b01829bea4c190ab0d8 2024-11-14T06:17:56,754 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/f70439a41cfb4b01829bea4c190ab0d8, entries=150, sequenceid=459, filesize=30.5 K 2024-11-14T06:17:56,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/16700f0f53ef4154b9f5a6eecb67a163 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/16700f0f53ef4154b9f5a6eecb67a163 2024-11-14T06:17:56,765 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/16700f0f53ef4154b9f5a6eecb67a163, entries=150, sequenceid=459, filesize=12.0 K 2024-11-14T06:17:56,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/6fa2c3a1aeb84813a336628daebfb6b2 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/6fa2c3a1aeb84813a336628daebfb6b2 2024-11-14T06:17:56,773 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/6fa2c3a1aeb84813a336628daebfb6b2, entries=150, sequenceid=459, filesize=12.0 K 2024-11-14T06:17:56,774 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 8b0928b43a2d9ea8c8bf8f2250ec54df in 965ms, sequenceid=459, compaction requested=false 2024-11-14T06:17:56,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 8b0928b43a2d9ea8c8bf8f2250ec54df: 2024-11-14T06:17:56,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:56,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-14T06:17:56,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-14T06:17:56,777 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-14T06:17:56,777 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2730 sec 2024-11-14T06:17:56,782 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.2830 sec 2024-11-14T06:17:57,195 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8b0928b43a2d9ea8c8bf8f2250ec54df 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-14T06:17:57,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b0928b43a2d9ea8c8bf8f2250ec54df, store=A 2024-11-14T06:17:57,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:57,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b0928b43a2d9ea8c8bf8f2250ec54df, store=B 2024-11-14T06:17:57,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:57,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b0928b43a2d9ea8c8bf8f2250ec54df, store=C 2024-11-14T06:17:57,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:57,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on 8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:57,227 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111465dfa5de41644eec987ed57731a300c9_8b0928b43a2d9ea8c8bf8f2250ec54df is 50, key is test_row_0/A:col10/1731565076008/Put/seqid=0 2024-11-14T06:17:57,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742260_1436 (size=14994) 2024-11-14T06:17:57,300 INFO [Thread-1609 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1609. 2024-11-14T06:17:57,300 DEBUG [Thread-1609 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T06:17:57,300 DEBUG [Thread-1609 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:17:57,300 DEBUG [Thread-1609 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:17:57,300 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:17:57,303 INFO [Thread-1598 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1598. 2024-11-14T06:17:57,303 DEBUG [Thread-1598 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T06:17:57,303 DEBUG [Thread-1598 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:17:57,303 DEBUG [Thread-1598 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:17:57,303 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:17:57,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:57,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57382 deadline: 1731565137293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:57,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:57,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:57,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:57,311 INFO [Thread-1607 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1607. 2024-11-14T06:17:57,311 DEBUG [Thread-1607 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T06:17:57,311 DEBUG [Thread-1607 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:17:57,312 DEBUG [Thread-1607 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:17:57,312 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:17:57,312 INFO [Thread-1601 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1601. 2024-11-14T06:17:57,312 DEBUG [Thread-1601 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T06:17:57,312 DEBUG [Thread-1601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:17:57,312 DEBUG [Thread-1601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:17:57,312 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:17:57,322 INFO [Thread-1603 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1603. 2024-11-14T06:17:57,322 DEBUG [Thread-1603 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T06:17:57,322 DEBUG [Thread-1603 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:17:57,322 DEBUG [Thread-1603 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:17:57,322 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:17:57,325 INFO [Thread-1605 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1605. 2024-11-14T06:17:57,325 DEBUG [Thread-1605 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T06:17:57,325 DEBUG [Thread-1605 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:17:57,325 DEBUG [Thread-1605 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:17:57,325 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:17:57,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:57,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57382 deadline: 1731565137418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:57,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:57,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:57,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:57,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:57,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57382 deadline: 1731565137628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:57,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:57,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:57,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:57,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-14T06:17:57,648 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:17:57,657 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:57,662 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111465dfa5de41644eec987ed57731a300c9_8b0928b43a2d9ea8c8bf8f2250ec54df to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111465dfa5de41644eec987ed57731a300c9_8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:57,662 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/a4eef96520e24027b033521b40582ab8, store: [table=TestAcidGuarantees family=A region=8b0928b43a2d9ea8c8bf8f2250ec54df] 2024-11-14T06:17:57,663 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/a4eef96520e24027b033521b40582ab8 is 175, key is test_row_0/A:col10/1731565076008/Put/seqid=0 2024-11-14T06:17:57,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742261_1437 (size=39949) 2024-11-14T06:17:57,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:57,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57382 deadline: 1731565137937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:57,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:57,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:57,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:58,066 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=479, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/a4eef96520e24027b033521b40582ab8 2024-11-14T06:17:58,072 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/b396e9fc30b64885a50598b5aa5ad9f5 is 50, key is test_row_0/B:col10/1731565076008/Put/seqid=0 2024-11-14T06:17:58,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742262_1438 (size=12301) 2024-11-14T06:17:58,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:17:58,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57382 deadline: 1731565138447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:17:58,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:58,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:17:58,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:17:58,482 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/b396e9fc30b64885a50598b5aa5ad9f5 2024-11-14T06:17:58,493 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/99f3bdbc749346bf9ae2f734ac64aad7 is 50, key is test_row_0/C:col10/1731565076008/Put/seqid=0 2024-11-14T06:17:58,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742263_1439 (size=12301) 2024-11-14T06:17:58,898 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/99f3bdbc749346bf9ae2f734ac64aad7 2024-11-14T06:17:58,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/a4eef96520e24027b033521b40582ab8 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/a4eef96520e24027b033521b40582ab8 2024-11-14T06:17:58,906 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/a4eef96520e24027b033521b40582ab8, entries=200, sequenceid=479, filesize=39.0 K 2024-11-14T06:17:58,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/b396e9fc30b64885a50598b5aa5ad9f5 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/b396e9fc30b64885a50598b5aa5ad9f5 2024-11-14T06:17:58,910 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/b396e9fc30b64885a50598b5aa5ad9f5, entries=150, sequenceid=479, filesize=12.0 K 2024-11-14T06:17:58,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/99f3bdbc749346bf9ae2f734ac64aad7 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/99f3bdbc749346bf9ae2f734ac64aad7 2024-11-14T06:17:58,915 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/99f3bdbc749346bf9ae2f734ac64aad7, entries=150, sequenceid=479, filesize=12.0 K 2024-11-14T06:17:58,915 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 8b0928b43a2d9ea8c8bf8f2250ec54df in 1720ms, sequenceid=479, compaction requested=true 2024-11-14T06:17:58,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8b0928b43a2d9ea8c8bf8f2250ec54df: 2024-11-14T06:17:58,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b0928b43a2d9ea8c8bf8f2250ec54df:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T06:17:58,916 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:17:58,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:58,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b0928b43a2d9ea8c8bf8f2250ec54df:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T06:17:58,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:58,916 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:17:58,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b0928b43a2d9ea8c8bf8f2250ec54df:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T06:17:58,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:17:58,916 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:17:58,916 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:17:58,916 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): 8b0928b43a2d9ea8c8bf8f2250ec54df/A is initiating minor compaction (all files) 2024-11-14T06:17:58,916 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): 8b0928b43a2d9ea8c8bf8f2250ec54df/B is initiating minor compaction (all files) 2024-11-14T06:17:58,916 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b0928b43a2d9ea8c8bf8f2250ec54df/A in TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:58,916 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b0928b43a2d9ea8c8bf8f2250ec54df/B in TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:58,916 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/c09579a40926430397bbb3d291f9ac2c, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/f70439a41cfb4b01829bea4c190ab0d8, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/a4eef96520e24027b033521b40582ab8] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp, totalSize=100.7 K 2024-11-14T06:17:58,916 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/c34ad7694753407d8251d66ed9f322af, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/16700f0f53ef4154b9f5a6eecb67a163, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/b396e9fc30b64885a50598b5aa5ad9f5] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp, totalSize=36.7 K 2024-11-14T06:17:58,916 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:58,916 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. files: [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/c09579a40926430397bbb3d291f9ac2c, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/f70439a41cfb4b01829bea4c190ab0d8, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/a4eef96520e24027b033521b40582ab8] 2024-11-14T06:17:58,917 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting c09579a40926430397bbb3d291f9ac2c, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1731565074832 2024-11-14T06:17:58,917 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting c34ad7694753407d8251d66ed9f322af, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1731565074832 2024-11-14T06:17:58,917 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting f70439a41cfb4b01829bea4c190ab0d8, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1731565075259 2024-11-14T06:17:58,917 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 16700f0f53ef4154b9f5a6eecb67a163, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1731565075259 2024-11-14T06:17:58,917 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4eef96520e24027b033521b40582ab8, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1731565076008 2024-11-14T06:17:58,917 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting b396e9fc30b64885a50598b5aa5ad9f5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1731565076008 2024-11-14T06:17:58,923 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8b0928b43a2d9ea8c8bf8f2250ec54df] 2024-11-14T06:17:58,924 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b0928b43a2d9ea8c8bf8f2250ec54df#B#compaction#369 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:17:58,925 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/bbeaf158d65c49ab9475efa6f01d60c8 is 50, key is test_row_0/B:col10/1731565076008/Put/seqid=0 2024-11-14T06:17:58,930 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411145a148093023347bfb671d763571b7455_8b0928b43a2d9ea8c8bf8f2250ec54df store=[table=TestAcidGuarantees family=A region=8b0928b43a2d9ea8c8bf8f2250ec54df] 2024-11-14T06:17:58,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742264_1440 (size=13051) 2024-11-14T06:17:58,933 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411145a148093023347bfb671d763571b7455_8b0928b43a2d9ea8c8bf8f2250ec54df, store=[table=TestAcidGuarantees family=A region=8b0928b43a2d9ea8c8bf8f2250ec54df] 2024-11-14T06:17:58,933 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411145a148093023347bfb671d763571b7455_8b0928b43a2d9ea8c8bf8f2250ec54df because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8b0928b43a2d9ea8c8bf8f2250ec54df] 2024-11-14T06:17:58,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742265_1441 (size=4469) 2024-11-14T06:17:59,336 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/bbeaf158d65c49ab9475efa6f01d60c8 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/bbeaf158d65c49ab9475efa6f01d60c8 2024-11-14T06:17:59,341 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b0928b43a2d9ea8c8bf8f2250ec54df/B of 8b0928b43a2d9ea8c8bf8f2250ec54df into bbeaf158d65c49ab9475efa6f01d60c8(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:17:59,341 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b0928b43a2d9ea8c8bf8f2250ec54df: 2024-11-14T06:17:59,341 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., storeName=8b0928b43a2d9ea8c8bf8f2250ec54df/B, priority=13, startTime=1731565078916; duration=0sec 2024-11-14T06:17:59,341 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:17:59,341 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b0928b43a2d9ea8c8bf8f2250ec54df:B 2024-11-14T06:17:59,341 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:17:59,342 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b0928b43a2d9ea8c8bf8f2250ec54df#A#compaction#370 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:17:59,342 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/be58e6ba94024eeab3f9b002bf51d9f6 is 175, key is test_row_0/A:col10/1731565076008/Put/seqid=0 2024-11-14T06:17:59,345 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:17:59,345 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): 8b0928b43a2d9ea8c8bf8f2250ec54df/C is initiating minor compaction (all files) 2024-11-14T06:17:59,345 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b0928b43a2d9ea8c8bf8f2250ec54df/C in TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:17:59,345 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/e2858f6ac6e24bafa8bae6ca99889ea5, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/6fa2c3a1aeb84813a336628daebfb6b2, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/99f3bdbc749346bf9ae2f734ac64aad7] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp, totalSize=36.7 K 2024-11-14T06:17:59,345 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting e2858f6ac6e24bafa8bae6ca99889ea5, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1731565074832 2024-11-14T06:17:59,345 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 6fa2c3a1aeb84813a336628daebfb6b2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1731565075259 2024-11-14T06:17:59,346 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 99f3bdbc749346bf9ae2f734ac64aad7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1731565076008 2024-11-14T06:17:59,359 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b0928b43a2d9ea8c8bf8f2250ec54df#C#compaction#371 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:17:59,360 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/e96ce597605746c0b539ac2ea45d02be is 50, key is test_row_0/C:col10/1731565076008/Put/seqid=0 2024-11-14T06:17:59,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742266_1442 (size=32005) 2024-11-14T06:17:59,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742267_1443 (size=13051) 2024-11-14T06:17:59,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on 8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:59,458 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8b0928b43a2d9ea8c8bf8f2250ec54df 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-14T06:17:59,458 INFO [Thread-1594 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1594. 2024-11-14T06:17:59,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b0928b43a2d9ea8c8bf8f2250ec54df, store=A 2024-11-14T06:17:59,458 DEBUG [Thread-1594 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T06:17:59,458 DEBUG [Thread-1594 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:17:59,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:59,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b0928b43a2d9ea8c8bf8f2250ec54df, store=B 2024-11-14T06:17:59,459 DEBUG [Thread-1594 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:17:59,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:59,459 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:17:59,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b0928b43a2d9ea8c8bf8f2250ec54df, store=C 2024-11-14T06:17:59,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:17:59,464 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114bc706b0772844d68af7f00fc32e1599f_8b0928b43a2d9ea8c8bf8f2250ec54df is 50, key is test_row_0/A:col10/1731565077280/Put/seqid=0 2024-11-14T06:17:59,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742268_1444 (size=12454) 2024-11-14T06:17:59,765 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/be58e6ba94024eeab3f9b002bf51d9f6 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/be58e6ba94024eeab3f9b002bf51d9f6 2024-11-14T06:17:59,769 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b0928b43a2d9ea8c8bf8f2250ec54df/A of 8b0928b43a2d9ea8c8bf8f2250ec54df into be58e6ba94024eeab3f9b002bf51d9f6(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:17:59,769 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b0928b43a2d9ea8c8bf8f2250ec54df: 2024-11-14T06:17:59,769 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., storeName=8b0928b43a2d9ea8c8bf8f2250ec54df/A, priority=13, startTime=1731565078915; duration=0sec 2024-11-14T06:17:59,769 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:59,769 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b0928b43a2d9ea8c8bf8f2250ec54df:A 2024-11-14T06:17:59,772 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/e96ce597605746c0b539ac2ea45d02be as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/e96ce597605746c0b539ac2ea45d02be 2024-11-14T06:17:59,776 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b0928b43a2d9ea8c8bf8f2250ec54df/C of 8b0928b43a2d9ea8c8bf8f2250ec54df into e96ce597605746c0b539ac2ea45d02be(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:17:59,777 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b0928b43a2d9ea8c8bf8f2250ec54df: 2024-11-14T06:17:59,777 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., storeName=8b0928b43a2d9ea8c8bf8f2250ec54df/C, priority=13, startTime=1731565078916; duration=0sec 2024-11-14T06:17:59,777 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:17:59,777 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b0928b43a2d9ea8c8bf8f2250ec54df:C 2024-11-14T06:17:59,868 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:17:59,872 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114bc706b0772844d68af7f00fc32e1599f_8b0928b43a2d9ea8c8bf8f2250ec54df to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114bc706b0772844d68af7f00fc32e1599f_8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:17:59,873 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/96a041f9ddbc4eab9b7d40eb4dc6e4a0, store: [table=TestAcidGuarantees family=A region=8b0928b43a2d9ea8c8bf8f2250ec54df] 2024-11-14T06:17:59,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/96a041f9ddbc4eab9b7d40eb4dc6e4a0 is 175, key is test_row_0/A:col10/1731565077280/Put/seqid=0 2024-11-14T06:17:59,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742269_1445 (size=31255) 2024-11-14T06:18:00,278 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=496, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/96a041f9ddbc4eab9b7d40eb4dc6e4a0 2024-11-14T06:18:00,283 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/31594c66667c46aba26c8543a780b273 is 50, key is test_row_0/B:col10/1731565077280/Put/seqid=0 2024-11-14T06:18:00,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742270_1446 (size=12301) 2024-11-14T06:18:00,687 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/31594c66667c46aba26c8543a780b273 2024-11-14T06:18:00,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/9338bc75bfdd4068b369e0ec8558da5a is 50, key is test_row_0/C:col10/1731565077280/Put/seqid=0 2024-11-14T06:18:00,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742271_1447 (size=12301) 2024-11-14T06:18:01,103 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/9338bc75bfdd4068b369e0ec8558da5a 2024-11-14T06:18:01,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/96a041f9ddbc4eab9b7d40eb4dc6e4a0 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/96a041f9ddbc4eab9b7d40eb4dc6e4a0 2024-11-14T06:18:01,118 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/96a041f9ddbc4eab9b7d40eb4dc6e4a0, entries=150, sequenceid=496, filesize=30.5 K 2024-11-14T06:18:01,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/31594c66667c46aba26c8543a780b273 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/31594c66667c46aba26c8543a780b273 2024-11-14T06:18:01,123 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/31594c66667c46aba26c8543a780b273, entries=150, sequenceid=496, filesize=12.0 K 2024-11-14T06:18:01,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/9338bc75bfdd4068b369e0ec8558da5a as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/9338bc75bfdd4068b369e0ec8558da5a 2024-11-14T06:18:01,128 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/9338bc75bfdd4068b369e0ec8558da5a, entries=150, sequenceid=496, filesize=12.0 K 2024-11-14T06:18:01,129 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=0 B/0 for 8b0928b43a2d9ea8c8bf8f2250ec54df in 1671ms, sequenceid=496, compaction requested=false 2024-11-14T06:18:01,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8b0928b43a2d9ea8c8bf8f2250ec54df: 2024-11-14T06:18:02,439 INFO [Thread-1592 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1592. 2024-11-14T06:18:02,439 INFO [Thread-1596 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1596. 2024-11-14T06:18:02,439 DEBUG [Thread-1592 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T06:18:02,439 DEBUG [Thread-1596 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T06:18:02,439 DEBUG [Thread-1592 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:02,439 DEBUG [Thread-1596 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:02,439 DEBUG [Thread-1592 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:02,439 DEBUG [Thread-1596 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:02,439 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:18:02,439 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:18:02,520 INFO [Thread-1590 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1590. 2024-11-14T06:18:02,520 DEBUG [Thread-1590 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T06:18:02,520 DEBUG [Thread-1590 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:02,521 DEBUG [Thread-1590 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:02,521 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:18:02,521 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T06:18:02,521 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 34 2024-11-14T06:18:02,521 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 28 2024-11-14T06:18:02,521 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 170 2024-11-14T06:18:02,521 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 28 2024-11-14T06:18:02,522 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 165 2024-11-14T06:18:02,522 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T06:18:02,522 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T06:18:02,522 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 919 2024-11-14T06:18:02,522 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2750 rows 2024-11-14T06:18:02,522 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 920 2024-11-14T06:18:02,522 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2759 rows 2024-11-14T06:18:02,522 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 923 2024-11-14T06:18:02,522 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2768 rows 2024-11-14T06:18:02,522 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 917 2024-11-14T06:18:02,522 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2747 rows 2024-11-14T06:18:02,522 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 925 2024-11-14T06:18:02,522 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2774 rows 2024-11-14T06:18:02,523 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T06:18:02,523 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T06:18:02,523 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:02,523 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:02,524 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:18:02,525 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-14T06:18:02,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T06:18:02,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-14T06:18:02,529 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731565082529"}]},"ts":"1731565082529"} 2024-11-14T06:18:02,531 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T06:18:02,531 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T06:18:02,532 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T06:18:02,534 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b0928b43a2d9ea8c8bf8f2250ec54df, UNASSIGN}] 2024-11-14T06:18:02,535 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b0928b43a2d9ea8c8bf8f2250ec54df, UNASSIGN 2024-11-14T06:18:02,536 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=8b0928b43a2d9ea8c8bf8f2250ec54df, regionState=CLOSING, regionLocation=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:02,537 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36045 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=CLOSING, location=ef93ef3a83c5,42283,1731564965945, table=TestAcidGuarantees, region=8b0928b43a2d9ea8c8bf8f2250ec54df. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-14T06:18:02,537 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b0928b43a2d9ea8c8bf8f2250ec54df, UNASSIGN because future has completed 2024-11-14T06:18:02,538 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T06:18:02,538 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945}] 2024-11-14T06:18:02,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-14T06:18:02,691 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(122): Close 8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:18:02,691 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T06:18:02,691 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1722): Closing 8b0928b43a2d9ea8c8bf8f2250ec54df, disabling compactions & flushes 2024-11-14T06:18:02,691 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:18:02,691 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:18:02,691 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. after waiting 0 ms 2024-11-14T06:18:02,691 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:18:02,692 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(2902): Flushing 8b0928b43a2d9ea8c8bf8f2250ec54df 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-14T06:18:02,692 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b0928b43a2d9ea8c8bf8f2250ec54df, store=A 2024-11-14T06:18:02,692 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:02,692 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b0928b43a2d9ea8c8bf8f2250ec54df, store=B 2024-11-14T06:18:02,692 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:02,692 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b0928b43a2d9ea8c8bf8f2250ec54df, store=C 2024-11-14T06:18:02,692 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:02,701 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114be7e994beea847249876dbce4b063b28_8b0928b43a2d9ea8c8bf8f2250ec54df is 50, key is test_row_1/A:col10/1731565082437/Put/seqid=0 2024-11-14T06:18:02,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742272_1448 (size=9914) 2024-11-14T06:18:02,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-14T06:18:03,107 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:18:03,115 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114be7e994beea847249876dbce4b063b28_8b0928b43a2d9ea8c8bf8f2250ec54df to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114be7e994beea847249876dbce4b063b28_8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:18:03,117 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/2b28216b63064890baddddbd04513312, store: [table=TestAcidGuarantees family=A region=8b0928b43a2d9ea8c8bf8f2250ec54df] 2024-11-14T06:18:03,117 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/2b28216b63064890baddddbd04513312 is 175, key is test_row_1/A:col10/1731565082437/Put/seqid=0 2024-11-14T06:18:03,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742273_1449 (size=22561) 2024-11-14T06:18:03,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-14T06:18:03,522 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=504, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/2b28216b63064890baddddbd04513312 2024-11-14T06:18:03,527 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/952585c6e19048739a8482a9b4cf64a7 is 50, key is test_row_1/B:col10/1731565082437/Put/seqid=0 2024-11-14T06:18:03,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742274_1450 (size=9857) 2024-11-14T06:18:03,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-14T06:18:03,932 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=504 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/952585c6e19048739a8482a9b4cf64a7 2024-11-14T06:18:03,939 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/579ef07d420c4b4ca2476fb43eb8f528 is 50, key is test_row_1/C:col10/1731565082437/Put/seqid=0 2024-11-14T06:18:03,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742275_1451 (size=9857) 2024-11-14T06:18:04,283 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T06:18:04,347 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=504 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/579ef07d420c4b4ca2476fb43eb8f528 2024-11-14T06:18:04,355 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/A/2b28216b63064890baddddbd04513312 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/2b28216b63064890baddddbd04513312 2024-11-14T06:18:04,359 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/2b28216b63064890baddddbd04513312, entries=100, sequenceid=504, filesize=22.0 K 2024-11-14T06:18:04,360 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/B/952585c6e19048739a8482a9b4cf64a7 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/952585c6e19048739a8482a9b4cf64a7 2024-11-14T06:18:04,363 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/952585c6e19048739a8482a9b4cf64a7, entries=100, sequenceid=504, filesize=9.6 K 2024-11-14T06:18:04,363 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/.tmp/C/579ef07d420c4b4ca2476fb43eb8f528 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/579ef07d420c4b4ca2476fb43eb8f528 2024-11-14T06:18:04,366 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/579ef07d420c4b4ca2476fb43eb8f528, entries=100, sequenceid=504, filesize=9.6 K 2024-11-14T06:18:04,367 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 8b0928b43a2d9ea8c8bf8f2250ec54df in 1676ms, sequenceid=504, compaction requested=true 2024-11-14T06:18:04,367 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/2aa41640c78144a38b1dba9f2f406934, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/2273c8cdaabf49118107593c6def0c72, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/e322e2c634a84ffa9a29945bc9b01b1e, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/539220dfdc0143119ec34ac676b14b8b, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/45a3933c9be9406e8c23e9c174fac46e, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/09c201694a6c4593b7d8eec439dce808, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/6cf9815e8cc3400ca646696169265d28, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/7c8477cd734b43c3b33634608b2a4b03, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/c753f9ecab0f4746a52328198c1bdeb5, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/64e8a2b69e1345cbbe95622b38cb4419, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/4600656d24c940e2a03255f83af728bd, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/57eca7bc2fdd4599be2dd308ce17083f, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/ffc309db89e64ccd86f743bf027453f1, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/ba4c2abe2e54438f91d0f9f743a69a09, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/1e38abdd38e249f08cb3f4f2b298ddd8, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/f4f1dd27e1d64ab18ce6c97b3b8d550c, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/aa9ea5ddfd684477a5918cdd0f2d7e7c, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/91f66ef144a0422cb80a24413449c73d, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/32bdf305fa6743a4bd9b9c48183ac2c2, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/c09579a40926430397bbb3d291f9ac2c, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/f70439a41cfb4b01829bea4c190ab0d8, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/a4eef96520e24027b033521b40582ab8] to archive 2024-11-14T06:18:04,368 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T06:18:04,369 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/2aa41640c78144a38b1dba9f2f406934 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/2aa41640c78144a38b1dba9f2f406934 2024-11-14T06:18:04,370 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/2273c8cdaabf49118107593c6def0c72 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/2273c8cdaabf49118107593c6def0c72 2024-11-14T06:18:04,371 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/e322e2c634a84ffa9a29945bc9b01b1e to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/e322e2c634a84ffa9a29945bc9b01b1e 2024-11-14T06:18:04,372 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/539220dfdc0143119ec34ac676b14b8b to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/539220dfdc0143119ec34ac676b14b8b 2024-11-14T06:18:04,372 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/45a3933c9be9406e8c23e9c174fac46e to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/45a3933c9be9406e8c23e9c174fac46e 2024-11-14T06:18:04,373 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/09c201694a6c4593b7d8eec439dce808 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/09c201694a6c4593b7d8eec439dce808 2024-11-14T06:18:04,374 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/6cf9815e8cc3400ca646696169265d28 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/6cf9815e8cc3400ca646696169265d28 2024-11-14T06:18:04,375 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/7c8477cd734b43c3b33634608b2a4b03 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/7c8477cd734b43c3b33634608b2a4b03 2024-11-14T06:18:04,375 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/c753f9ecab0f4746a52328198c1bdeb5 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/c753f9ecab0f4746a52328198c1bdeb5 2024-11-14T06:18:04,376 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/64e8a2b69e1345cbbe95622b38cb4419 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/64e8a2b69e1345cbbe95622b38cb4419 2024-11-14T06:18:04,377 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/4600656d24c940e2a03255f83af728bd to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/4600656d24c940e2a03255f83af728bd 2024-11-14T06:18:04,378 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/57eca7bc2fdd4599be2dd308ce17083f to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/57eca7bc2fdd4599be2dd308ce17083f 2024-11-14T06:18:04,378 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/ffc309db89e64ccd86f743bf027453f1 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/ffc309db89e64ccd86f743bf027453f1 2024-11-14T06:18:04,379 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/ba4c2abe2e54438f91d0f9f743a69a09 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/ba4c2abe2e54438f91d0f9f743a69a09 2024-11-14T06:18:04,380 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/1e38abdd38e249f08cb3f4f2b298ddd8 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/1e38abdd38e249f08cb3f4f2b298ddd8 2024-11-14T06:18:04,380 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/f4f1dd27e1d64ab18ce6c97b3b8d550c to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/f4f1dd27e1d64ab18ce6c97b3b8d550c 2024-11-14T06:18:04,381 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/aa9ea5ddfd684477a5918cdd0f2d7e7c to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/aa9ea5ddfd684477a5918cdd0f2d7e7c 2024-11-14T06:18:04,382 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/91f66ef144a0422cb80a24413449c73d to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/91f66ef144a0422cb80a24413449c73d 2024-11-14T06:18:04,383 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/32bdf305fa6743a4bd9b9c48183ac2c2 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/32bdf305fa6743a4bd9b9c48183ac2c2 2024-11-14T06:18:04,383 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/c09579a40926430397bbb3d291f9ac2c to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/c09579a40926430397bbb3d291f9ac2c 2024-11-14T06:18:04,384 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/f70439a41cfb4b01829bea4c190ab0d8 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/f70439a41cfb4b01829bea4c190ab0d8 2024-11-14T06:18:04,385 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/a4eef96520e24027b033521b40582ab8 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/a4eef96520e24027b033521b40582ab8 2024-11-14T06:18:04,386 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/35d1c9ef29eb43df96016ebc9e8308a4, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/f8bd7fab0aca4689a95aebea33bfb425, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/cd35d4671f1e425797703c530aa8cb7a, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/7fe0c4dff8914de99221dee4f7927e85, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/7c04561c234c444cb5eda7cc00b0d2d2, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/4bd1540165324bc99d42db9538746c13, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/7b6be02cd67745608859af71a22068ca, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/0301544542c04e80b9af9a72d37d278d, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/08030e0610ea4d8ebd4e70c1b86a274c, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/05d91ff6c48348c8be5a959a1a28ccbf, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/b03e6fd736174f0f868fe2d17133b9ec, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/9f22a5190d814aa08dc1ba28933c14f4, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/0827ebfe636341ae94af72e7957644a8, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/ba33d4fe45674a568ea02cb13336c63c, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/f7f5754e0eb1423fadefd9015041b0d9, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/459a4d5233104415b82fad0469d32e9c, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/b074ebc37ced4705be9c8aa9d27e2aa2, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/1ed853bc229c48d88d270aaed7f303cf, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/c34ad7694753407d8251d66ed9f322af, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/11137ea950ae4b62a1f9cfe195cb9d3b, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/16700f0f53ef4154b9f5a6eecb67a163, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/b396e9fc30b64885a50598b5aa5ad9f5] to archive 2024-11-14T06:18:04,387 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T06:18:04,388 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/35d1c9ef29eb43df96016ebc9e8308a4 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/35d1c9ef29eb43df96016ebc9e8308a4 2024-11-14T06:18:04,389 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/f8bd7fab0aca4689a95aebea33bfb425 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/f8bd7fab0aca4689a95aebea33bfb425 2024-11-14T06:18:04,390 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/cd35d4671f1e425797703c530aa8cb7a to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/cd35d4671f1e425797703c530aa8cb7a 2024-11-14T06:18:04,390 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/7fe0c4dff8914de99221dee4f7927e85 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/7fe0c4dff8914de99221dee4f7927e85 2024-11-14T06:18:04,391 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/7c04561c234c444cb5eda7cc00b0d2d2 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/7c04561c234c444cb5eda7cc00b0d2d2 2024-11-14T06:18:04,392 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/4bd1540165324bc99d42db9538746c13 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/4bd1540165324bc99d42db9538746c13 2024-11-14T06:18:04,393 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/7b6be02cd67745608859af71a22068ca to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/7b6be02cd67745608859af71a22068ca 2024-11-14T06:18:04,393 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/0301544542c04e80b9af9a72d37d278d to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/0301544542c04e80b9af9a72d37d278d 2024-11-14T06:18:04,394 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/08030e0610ea4d8ebd4e70c1b86a274c to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/08030e0610ea4d8ebd4e70c1b86a274c 2024-11-14T06:18:04,395 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/05d91ff6c48348c8be5a959a1a28ccbf to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/05d91ff6c48348c8be5a959a1a28ccbf 2024-11-14T06:18:04,396 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/b03e6fd736174f0f868fe2d17133b9ec to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/b03e6fd736174f0f868fe2d17133b9ec 2024-11-14T06:18:04,396 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/9f22a5190d814aa08dc1ba28933c14f4 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/9f22a5190d814aa08dc1ba28933c14f4 2024-11-14T06:18:04,397 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/0827ebfe636341ae94af72e7957644a8 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/0827ebfe636341ae94af72e7957644a8 2024-11-14T06:18:04,398 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/ba33d4fe45674a568ea02cb13336c63c to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/ba33d4fe45674a568ea02cb13336c63c 2024-11-14T06:18:04,398 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/f7f5754e0eb1423fadefd9015041b0d9 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/f7f5754e0eb1423fadefd9015041b0d9 2024-11-14T06:18:04,399 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/459a4d5233104415b82fad0469d32e9c to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/459a4d5233104415b82fad0469d32e9c 2024-11-14T06:18:04,400 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/b074ebc37ced4705be9c8aa9d27e2aa2 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/b074ebc37ced4705be9c8aa9d27e2aa2 2024-11-14T06:18:04,400 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/1ed853bc229c48d88d270aaed7f303cf to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/1ed853bc229c48d88d270aaed7f303cf 2024-11-14T06:18:04,401 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/c34ad7694753407d8251d66ed9f322af to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/c34ad7694753407d8251d66ed9f322af 2024-11-14T06:18:04,402 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/11137ea950ae4b62a1f9cfe195cb9d3b to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/11137ea950ae4b62a1f9cfe195cb9d3b 2024-11-14T06:18:04,402 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/16700f0f53ef4154b9f5a6eecb67a163 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/16700f0f53ef4154b9f5a6eecb67a163 2024-11-14T06:18:04,403 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/b396e9fc30b64885a50598b5aa5ad9f5 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/b396e9fc30b64885a50598b5aa5ad9f5 2024-11-14T06:18:04,404 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/4d370e25165a4225b2e2b770dc5cba02, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/35915e179cf842ce97d461a72b584c2a, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/625d0c349d884ef8916b69c31da77a1c, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/1860b759719b4bc79f5d3d7346a4d497, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/92458d141a2042aa8031dea882bf3ec2, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/7199b188f87b49c4b60ef80872478f12, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/ff1f0d33941d41d5aa00da668029d4d0, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/deaf8ce985cf420284f561361b8e2c11, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/2fb8fb99f2da46a58a0da12ec59f557d, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/d92ef1adaa50417a8887871651cb9682, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/1576fb66e1fb4fb9b167c21a7aaa58cb, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/3dc7ced41413442bba190c64c2b5a089, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/313bf7bdef2745928d0c51106cd67b1d, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/5266699cf5124e80afc4206def8e3e0d, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/e9e1048230e94d1db057b0be79fb80dd, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/727036d956664ae3b4b14334c0122ddc, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/9d984f94cf274ecdb4303d3a752ef71d, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/972b9c490a764fc5ae7421dc9b7c8197, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/e2858f6ac6e24bafa8bae6ca99889ea5, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/4c7ca30a91164711bb23350304e79c5d, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/6fa2c3a1aeb84813a336628daebfb6b2, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/99f3bdbc749346bf9ae2f734ac64aad7] to archive 2024-11-14T06:18:04,405 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T06:18:04,406 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/4d370e25165a4225b2e2b770dc5cba02 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/4d370e25165a4225b2e2b770dc5cba02 2024-11-14T06:18:04,407 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/35915e179cf842ce97d461a72b584c2a to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/35915e179cf842ce97d461a72b584c2a 2024-11-14T06:18:04,408 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/625d0c349d884ef8916b69c31da77a1c to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/625d0c349d884ef8916b69c31da77a1c 2024-11-14T06:18:04,409 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/1860b759719b4bc79f5d3d7346a4d497 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/1860b759719b4bc79f5d3d7346a4d497 2024-11-14T06:18:04,410 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/92458d141a2042aa8031dea882bf3ec2 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/92458d141a2042aa8031dea882bf3ec2 2024-11-14T06:18:04,411 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/7199b188f87b49c4b60ef80872478f12 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/7199b188f87b49c4b60ef80872478f12 2024-11-14T06:18:04,412 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/ff1f0d33941d41d5aa00da668029d4d0 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/ff1f0d33941d41d5aa00da668029d4d0 2024-11-14T06:18:04,413 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/deaf8ce985cf420284f561361b8e2c11 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/deaf8ce985cf420284f561361b8e2c11 2024-11-14T06:18:04,414 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/2fb8fb99f2da46a58a0da12ec59f557d to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/2fb8fb99f2da46a58a0da12ec59f557d 2024-11-14T06:18:04,415 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/d92ef1adaa50417a8887871651cb9682 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/d92ef1adaa50417a8887871651cb9682 2024-11-14T06:18:04,416 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/1576fb66e1fb4fb9b167c21a7aaa58cb to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/1576fb66e1fb4fb9b167c21a7aaa58cb 2024-11-14T06:18:04,417 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/3dc7ced41413442bba190c64c2b5a089 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/3dc7ced41413442bba190c64c2b5a089 2024-11-14T06:18:04,418 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/313bf7bdef2745928d0c51106cd67b1d to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/313bf7bdef2745928d0c51106cd67b1d 2024-11-14T06:18:04,419 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/5266699cf5124e80afc4206def8e3e0d to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/5266699cf5124e80afc4206def8e3e0d 2024-11-14T06:18:04,420 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/e9e1048230e94d1db057b0be79fb80dd to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/e9e1048230e94d1db057b0be79fb80dd 2024-11-14T06:18:04,421 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/727036d956664ae3b4b14334c0122ddc to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/727036d956664ae3b4b14334c0122ddc 2024-11-14T06:18:04,423 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/9d984f94cf274ecdb4303d3a752ef71d to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/9d984f94cf274ecdb4303d3a752ef71d 2024-11-14T06:18:04,423 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/972b9c490a764fc5ae7421dc9b7c8197 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/972b9c490a764fc5ae7421dc9b7c8197 2024-11-14T06:18:04,424 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/e2858f6ac6e24bafa8bae6ca99889ea5 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/e2858f6ac6e24bafa8bae6ca99889ea5 2024-11-14T06:18:04,425 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/4c7ca30a91164711bb23350304e79c5d to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/4c7ca30a91164711bb23350304e79c5d 2024-11-14T06:18:04,426 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/6fa2c3a1aeb84813a336628daebfb6b2 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/6fa2c3a1aeb84813a336628daebfb6b2 2024-11-14T06:18:04,427 DEBUG [StoreCloser-TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/99f3bdbc749346bf9ae2f734ac64aad7 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/99f3bdbc749346bf9ae2f734ac64aad7 2024-11-14T06:18:04,431 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/recovered.edits/507.seqid, newMaxSeqId=507, maxSeqId=4 2024-11-14T06:18:04,432 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. 2024-11-14T06:18:04,432 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1676): Region close journal for 8b0928b43a2d9ea8c8bf8f2250ec54df: Waiting for close lock at 1731565082691Running coprocessor pre-close hooks at 1731565082691Disabling compacts and flushes for region at 1731565082691Disabling writes for close at 1731565082691Obtaining lock to block concurrent updates at 1731565082692 (+1 ms)Preparing flush snapshotting stores in 8b0928b43a2d9ea8c8bf8f2250ec54df at 1731565082692Finished memstore snapshotting TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1731565082692Flushing stores of TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df. at 1731565082693 (+1 ms)Flushing 8b0928b43a2d9ea8c8bf8f2250ec54df/A: creating writer at 1731565082693Flushing mob file 8b0928b43a2d9ea8c8bf8f2250ec54df/A: appending metadata at 1731565082701 (+8 ms)Flushing mob file 8b0928b43a2d9ea8c8bf8f2250ec54df/A: closing flushed file at 1731565082701Flushing 8b0928b43a2d9ea8c8bf8f2250ec54df/A: appending metadata at 1731565083117 (+416 ms)Flushing 8b0928b43a2d9ea8c8bf8f2250ec54df/A: closing flushed file at 1731565083117Flushing 8b0928b43a2d9ea8c8bf8f2250ec54df/B: creating writer at 1731565083525 (+408 ms)Flushing 8b0928b43a2d9ea8c8bf8f2250ec54df/B: appending metadata at 1731565083527 (+2 ms)Flushing 8b0928b43a2d9ea8c8bf8f2250ec54df/B: closing flushed file at 1731565083527Flushing 8b0928b43a2d9ea8c8bf8f2250ec54df/C: creating writer at 1731565083936 (+409 ms)Flushing 8b0928b43a2d9ea8c8bf8f2250ec54df/C: appending metadata at 1731565083938 (+2 ms)Flushing 8b0928b43a2d9ea8c8bf8f2250ec54df/C: closing flushed file at 1731565083938Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2c037246: reopening flushed file at 1731565084354 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@22504281: reopening flushed file at 1731565084359 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3a95c33a: reopening flushed file at 1731565084363 (+4 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 8b0928b43a2d9ea8c8bf8f2250ec54df in 1676ms, sequenceid=504, compaction requested=true at 1731565084367 (+4 ms)Writing region close event to WAL at 1731565084428 (+61 ms)Running coprocessor post-close hooks at 1731565084432 (+4 ms)Closed at 1731565084432 2024-11-14T06:18:04,434 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(157): Closed 8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:18:04,434 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=8b0928b43a2d9ea8c8bf8f2250ec54df, regionState=CLOSED 2024-11-14T06:18:04,436 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=114, ppid=113, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 because future has completed 2024-11-14T06:18:04,438 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-14T06:18:04,438 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; CloseRegionProcedure 8b0928b43a2d9ea8c8bf8f2250ec54df, server=ef93ef3a83c5,42283,1731564965945 in 1.8980 sec 2024-11-14T06:18:04,440 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=113, resume processing ppid=112 2024-11-14T06:18:04,440 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, ppid=112, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b0928b43a2d9ea8c8bf8f2250ec54df, UNASSIGN in 1.9040 sec 2024-11-14T06:18:04,442 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-14T06:18:04,442 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9080 sec 2024-11-14T06:18:04,443 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731565084443"}]},"ts":"1731565084443"} 2024-11-14T06:18:04,444 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T06:18:04,444 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T06:18:04,445 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9190 sec 2024-11-14T06:18:04,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-14T06:18:04,688 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:18:04,688 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-14T06:18:04,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T06:18:04,690 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=115, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T06:18:04,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-14T06:18:04,690 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=115, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T06:18:04,692 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:18:04,694 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A, FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B, FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C, FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/recovered.edits] 2024-11-14T06:18:04,697 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/2b28216b63064890baddddbd04513312 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/2b28216b63064890baddddbd04513312 2024-11-14T06:18:04,698 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/96a041f9ddbc4eab9b7d40eb4dc6e4a0 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/96a041f9ddbc4eab9b7d40eb4dc6e4a0 2024-11-14T06:18:04,699 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/be58e6ba94024eeab3f9b002bf51d9f6 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/A/be58e6ba94024eeab3f9b002bf51d9f6 2024-11-14T06:18:04,702 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/31594c66667c46aba26c8543a780b273 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/31594c66667c46aba26c8543a780b273 2024-11-14T06:18:04,703 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/952585c6e19048739a8482a9b4cf64a7 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/952585c6e19048739a8482a9b4cf64a7 2024-11-14T06:18:04,704 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/bbeaf158d65c49ab9475efa6f01d60c8 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/B/bbeaf158d65c49ab9475efa6f01d60c8 2024-11-14T06:18:04,706 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/579ef07d420c4b4ca2476fb43eb8f528 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/579ef07d420c4b4ca2476fb43eb8f528 2024-11-14T06:18:04,708 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/9338bc75bfdd4068b369e0ec8558da5a to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/9338bc75bfdd4068b369e0ec8558da5a 2024-11-14T06:18:04,709 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/e96ce597605746c0b539ac2ea45d02be to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/C/e96ce597605746c0b539ac2ea45d02be 2024-11-14T06:18:04,712 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/recovered.edits/507.seqid to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df/recovered.edits/507.seqid 2024-11-14T06:18:04,712 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:18:04,713 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T06:18:04,713 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T06:18:04,714 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-14T06:18:04,716 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114087cf6bb8ac84f9ab54a4815ab59e629_8b0928b43a2d9ea8c8bf8f2250ec54df to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114087cf6bb8ac84f9ab54a4815ab59e629_8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:18:04,717 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411140ea8750752a74a13b38ac2c3300b7160_8b0928b43a2d9ea8c8bf8f2250ec54df to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411140ea8750752a74a13b38ac2c3300b7160_8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:18:04,718 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111433eb7d425e334b248f343496aa6fc994_8b0928b43a2d9ea8c8bf8f2250ec54df to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111433eb7d425e334b248f343496aa6fc994_8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:18:04,719 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144ff1ef5c1fa342e3bf727c3e7d80ccbb_8b0928b43a2d9ea8c8bf8f2250ec54df to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144ff1ef5c1fa342e3bf727c3e7d80ccbb_8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:18:04,720 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114563a9a1c40c942968c8be1dd3dbbc00a_8b0928b43a2d9ea8c8bf8f2250ec54df to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114563a9a1c40c942968c8be1dd3dbbc00a_8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:18:04,721 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111465dfa5de41644eec987ed57731a300c9_8b0928b43a2d9ea8c8bf8f2250ec54df to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111465dfa5de41644eec987ed57731a300c9_8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:18:04,722 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411146931776b164d406fb5b5c8c50418d8c7_8b0928b43a2d9ea8c8bf8f2250ec54df to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411146931776b164d406fb5b5c8c50418d8c7_8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:18:04,723 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111473141ce1d49a403ca5bfba6849e50777_8b0928b43a2d9ea8c8bf8f2250ec54df to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111473141ce1d49a403ca5bfba6849e50777_8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:18:04,725 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114789d67529e1e4969bb1f8b8780019d61_8b0928b43a2d9ea8c8bf8f2250ec54df to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114789d67529e1e4969bb1f8b8780019d61_8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:18:04,726 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411147a1c1cbbeea14c6ab7d0b689e4e9ac58_8b0928b43a2d9ea8c8bf8f2250ec54df to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411147a1c1cbbeea14c6ab7d0b689e4e9ac58_8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:18:04,727 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411148745ac0a46cb4d78b6b74dc5ef14ca42_8b0928b43a2d9ea8c8bf8f2250ec54df to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411148745ac0a46cb4d78b6b74dc5ef14ca42_8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:18:04,728 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411148af17cd9177a416eac9fe70bbffeb4f1_8b0928b43a2d9ea8c8bf8f2250ec54df to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411148af17cd9177a416eac9fe70bbffeb4f1_8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:18:04,729 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114af30a4b97b5747ee970456e94f7b3c9c_8b0928b43a2d9ea8c8bf8f2250ec54df to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114af30a4b97b5747ee970456e94f7b3c9c_8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:18:04,730 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114bc706b0772844d68af7f00fc32e1599f_8b0928b43a2d9ea8c8bf8f2250ec54df to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114bc706b0772844d68af7f00fc32e1599f_8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:18:04,731 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114be7e994beea847249876dbce4b063b28_8b0928b43a2d9ea8c8bf8f2250ec54df to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114be7e994beea847249876dbce4b063b28_8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:18:04,732 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d23c922396674932aa3e9e556fac2262_8b0928b43a2d9ea8c8bf8f2250ec54df to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d23c922396674932aa3e9e556fac2262_8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:18:04,733 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d3bda702e063492e93caf9c393f3b513_8b0928b43a2d9ea8c8bf8f2250ec54df to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d3bda702e063492e93caf9c393f3b513_8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:18:04,734 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e74e176aa20b4674ad0e17f4bf74a13c_8b0928b43a2d9ea8c8bf8f2250ec54df to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e74e176aa20b4674ad0e17f4bf74a13c_8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:18:04,734 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114fd8187bfe26d480491ea1d285e05b535_8b0928b43a2d9ea8c8bf8f2250ec54df to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114fd8187bfe26d480491ea1d285e05b535_8b0928b43a2d9ea8c8bf8f2250ec54df 2024-11-14T06:18:04,735 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T06:18:04,736 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=115, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T06:18:04,738 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T06:18:04,739 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T06:18:04,740 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=115, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T06:18:04,740 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T06:18:04,740 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731565084740"}]},"ts":"9223372036854775807"} 2024-11-14T06:18:04,741 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T06:18:04,741 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 8b0928b43a2d9ea8c8bf8f2250ec54df, NAME => 'TestAcidGuarantees,,1731565054146.8b0928b43a2d9ea8c8bf8f2250ec54df.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T06:18:04,741 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T06:18:04,741 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731565084741"}]},"ts":"9223372036854775807"} 2024-11-14T06:18:04,742 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T06:18:04,742 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=115, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T06:18:04,743 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 54 msec 2024-11-14T06:18:04,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-14T06:18:04,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T06:18:04,800 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:18:04,808 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=249 (was 248) - Thread LEAK? -, OpenFileDescriptor=455 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=731 (was 854), ProcessCount=12 (was 11) - ProcessCount LEAK? -, AvailableMemoryMB=817 (was 242) - AvailableMemoryMB LEAK? - 2024-11-14T06:18:04,818 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=249, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=731, ProcessCount=11, AvailableMemoryMB=815 2024-11-14T06:18:04,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T06:18:04,819 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T06:18:04,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=116, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T06:18:04,821 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T06:18:04,821 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:18:04,821 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 116 2024-11-14T06:18:04,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-14T06:18:04,822 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T06:18:04,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742276_1452 (size=1061) 2024-11-14T06:18:04,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-14T06:18:05,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-14T06:18:05,229 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02 2024-11-14T06:18:05,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742277_1453 (size=53) 2024-11-14T06:18:05,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-14T06:18:05,538 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-14T06:18:05,640 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T06:18:05,640 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing c151951bebfcfd871997dfedda0e3149, disabling compactions & flushes 2024-11-14T06:18:05,640 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:05,640 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:05,640 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. after waiting 0 ms 2024-11-14T06:18:05,640 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:05,640 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:05,641 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for c151951bebfcfd871997dfedda0e3149: Waiting for close lock at 1731565085640Disabling compacts and flushes for region at 1731565085640Disabling writes for close at 1731565085640Writing region close event to WAL at 1731565085640Closed at 1731565085640 2024-11-14T06:18:05,643 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T06:18:05,643 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731565085643"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731565085643"}]},"ts":"1731565085643"} 2024-11-14T06:18:05,645 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T06:18:05,646 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T06:18:05,647 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731565085646"}]},"ts":"1731565085646"} 2024-11-14T06:18:05,649 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T06:18:05,650 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c151951bebfcfd871997dfedda0e3149, ASSIGN}] 2024-11-14T06:18:05,651 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c151951bebfcfd871997dfedda0e3149, ASSIGN 2024-11-14T06:18:05,652 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c151951bebfcfd871997dfedda0e3149, ASSIGN; state=OFFLINE, location=ef93ef3a83c5,42283,1731564965945; forceNewPlan=false, retain=false 2024-11-14T06:18:05,803 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=c151951bebfcfd871997dfedda0e3149, regionState=OPENING, regionLocation=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:05,805 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c151951bebfcfd871997dfedda0e3149, ASSIGN because future has completed 2024-11-14T06:18:05,805 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; OpenRegionProcedure c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945}] 2024-11-14T06:18:05,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-14T06:18:05,962 INFO [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:05,962 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7752): Opening region: {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} 2024-11-14T06:18:05,964 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c151951bebfcfd871997dfedda0e3149 2024-11-14T06:18:05,964 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T06:18:05,964 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7794): checking encryption for c151951bebfcfd871997dfedda0e3149 2024-11-14T06:18:05,964 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7797): checking classloading for c151951bebfcfd871997dfedda0e3149 2024-11-14T06:18:05,966 INFO [StoreOpener-c151951bebfcfd871997dfedda0e3149-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c151951bebfcfd871997dfedda0e3149 2024-11-14T06:18:05,968 INFO [StoreOpener-c151951bebfcfd871997dfedda0e3149-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T06:18:05,969 INFO [StoreOpener-c151951bebfcfd871997dfedda0e3149-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c151951bebfcfd871997dfedda0e3149 columnFamilyName A 2024-11-14T06:18:05,969 DEBUG [StoreOpener-c151951bebfcfd871997dfedda0e3149-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:18:05,969 INFO [StoreOpener-c151951bebfcfd871997dfedda0e3149-1 {}] regionserver.HStore(327): Store=c151951bebfcfd871997dfedda0e3149/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T06:18:05,970 INFO [StoreOpener-c151951bebfcfd871997dfedda0e3149-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c151951bebfcfd871997dfedda0e3149 2024-11-14T06:18:05,971 INFO [StoreOpener-c151951bebfcfd871997dfedda0e3149-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T06:18:05,972 INFO [StoreOpener-c151951bebfcfd871997dfedda0e3149-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c151951bebfcfd871997dfedda0e3149 columnFamilyName B 2024-11-14T06:18:05,972 DEBUG [StoreOpener-c151951bebfcfd871997dfedda0e3149-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:18:05,972 INFO [StoreOpener-c151951bebfcfd871997dfedda0e3149-1 {}] regionserver.HStore(327): Store=c151951bebfcfd871997dfedda0e3149/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T06:18:05,972 INFO [StoreOpener-c151951bebfcfd871997dfedda0e3149-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c151951bebfcfd871997dfedda0e3149 2024-11-14T06:18:05,973 INFO [StoreOpener-c151951bebfcfd871997dfedda0e3149-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T06:18:05,973 INFO [StoreOpener-c151951bebfcfd871997dfedda0e3149-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c151951bebfcfd871997dfedda0e3149 columnFamilyName C 2024-11-14T06:18:05,973 DEBUG [StoreOpener-c151951bebfcfd871997dfedda0e3149-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:18:05,973 INFO [StoreOpener-c151951bebfcfd871997dfedda0e3149-1 {}] regionserver.HStore(327): Store=c151951bebfcfd871997dfedda0e3149/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T06:18:05,974 INFO [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:05,974 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1038): replaying wal for c151951bebfcfd871997dfedda0e3149 2024-11-14T06:18:05,974 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149 2024-11-14T06:18:05,974 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149 2024-11-14T06:18:05,975 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1048): stopping wal replay for c151951bebfcfd871997dfedda0e3149 2024-11-14T06:18:05,975 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1060): Cleaning up temporary data for c151951bebfcfd871997dfedda0e3149 2024-11-14T06:18:05,975 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T06:18:05,976 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1093): writing seq id for c151951bebfcfd871997dfedda0e3149 2024-11-14T06:18:05,977 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T06:18:05,978 INFO [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1114): Opened c151951bebfcfd871997dfedda0e3149; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61685681, jitterRate=-0.08081172406673431}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T06:18:05,978 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c151951bebfcfd871997dfedda0e3149 2024-11-14T06:18:05,978 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1006): Region open journal for c151951bebfcfd871997dfedda0e3149: Running coprocessor pre-open hook at 1731565085964Writing region info on filesystem at 1731565085964Initializing all the Stores at 1731565085965 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731565085966 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731565085966Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731565085966Cleaning up temporary data from old regions at 1731565085975 (+9 ms)Running coprocessor post-open hooks at 1731565085978 (+3 ms)Region opened successfully at 1731565085978 2024-11-14T06:18:05,979 INFO [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., pid=118, masterSystemTime=1731565085957 2024-11-14T06:18:05,980 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:05,980 INFO [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:05,980 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=c151951bebfcfd871997dfedda0e3149, regionState=OPEN, openSeqNum=2, regionLocation=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:05,981 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=118, ppid=117, state=RUNNABLE, hasLock=false; OpenRegionProcedure c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 because future has completed 2024-11-14T06:18:05,983 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-14T06:18:05,983 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; OpenRegionProcedure c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 in 177 msec 2024-11-14T06:18:05,984 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=117, resume processing ppid=116 2024-11-14T06:18:05,984 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, ppid=116, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c151951bebfcfd871997dfedda0e3149, ASSIGN in 334 msec 2024-11-14T06:18:05,985 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T06:18:05,985 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731565085985"}]},"ts":"1731565085985"} 2024-11-14T06:18:05,987 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T06:18:05,988 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T06:18:05,989 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1690 sec 2024-11-14T06:18:06,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-14T06:18:06,968 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:18:06,970 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21351805, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:06,970 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:18:06,970 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:18:06,970 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:18:06,971 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:18:06,971 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:18:06,971 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@345bebf1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:06,971 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:18:06,971 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:18:06,971 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:06,972 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51552, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:18:06,972 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4349ff94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:06,972 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:18:06,973 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:18:06,973 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:18:06,974 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47578, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:18:06,975 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T06:18:06,976 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is ef93ef3a83c5,36045,1731564965170 2024-11-14T06:18:06,976 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@30794997 2024-11-14T06:18:06,976 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T06:18:06,977 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51556, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T06:18:06,978 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68981673, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:06,978 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:18:06,978 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:18:06,979 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:18:06,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:18:06,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:18:06,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39770f34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:06,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:18:06,980 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:18:06,980 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:06,980 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51562, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:18:06,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1118ce4a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:06,981 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e4d4669, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:06,981 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:18:06,981 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:18:06,982 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:18:06,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:18:06,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:18:06,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59e1681f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:06,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:18:06,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:18:06,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:06,983 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51592, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:18:06,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@497ba834, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:06,984 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@604b6096, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:06,984 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:18:06,984 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:18:06,985 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:18:06,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:18:06,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:18:06,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7817f308, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:06,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:18:06,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:18:06,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:06,986 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51608, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:18:06,986 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33c5d3ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:06,987 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bb5d48c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:06,987 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:18:06,987 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:18:06,987 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:18:06,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:18:06,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:18:06,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21ad6376, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:06,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:18:06,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:18:06,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:06,988 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51632, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:18:06,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29802d40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:06,989 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ca644d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:06,989 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:18:06,989 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:18:06,990 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:18:06,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:18:06,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:18:06,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79aa110f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:06,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:18:06,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:18:06,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:06,991 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51664, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:18:06,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71b179f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:06,992 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5758cf8e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:06,992 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:18:06,992 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:18:06,992 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:18:06,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:18:06,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:18:06,993 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c77342c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:06,993 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:18:06,993 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:18:06,993 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:06,993 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51682, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:18:06,994 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f3f6850, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:06,994 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b298682, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:06,994 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:18:06,995 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:18:06,995 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:18:06,995 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:18:06,995 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:18:06,996 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b244744, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:06,996 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:18:06,996 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:18:06,996 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:06,997 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51698, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:18:06,998 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f787b0e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:06,999 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f4765bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:06,999 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:18:06,999 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:18:06,999 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:18:07,000 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:18:07,000 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:18:07,000 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4745c5a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:07,000 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:18:07,000 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:18:07,000 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:07,001 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51730, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:18:07,001 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57c01a0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:07,002 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6deb2284, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:07,002 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:18:07,002 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:18:07,002 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:18:07,002 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:18:07,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:18:07,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44062b04, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:07,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:18:07,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:18:07,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:07,003 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51754, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:18:07,004 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21d73e06, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:07,004 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@197745a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:07,004 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:18:07,004 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:18:07,005 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:18:07,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:18:07,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:18:07,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@debf592, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:07,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:18:07,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:18:07,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:07,006 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51770, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:18:07,006 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13963ee9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:07,007 DEBUG [Thread-2030 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:18:07,007 DEBUG [Thread-2043 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:18:07,007 DEBUG [Thread-2041 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:18:07,007 DEBUG [Thread-2026 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:18:07,008 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:18:07,008 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:18:07,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:18:07,008 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:18:07,008 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:18:07,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:18:07,009 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:18:07,009 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:18:07,009 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47582, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:18:07,009 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47588, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:18:07,009 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47590, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:18:07,009 DEBUG [Thread-2045 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:18:07,009 DEBUG [Thread-2032 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:18:07,009 DEBUG [Thread-2039 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:18:07,010 DEBUG [Thread-2028 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:18:07,010 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47606, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:18:07,010 DEBUG [Thread-2037 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:18:07,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:18:07,010 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:18:07,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:18:07,010 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:18:07,011 DEBUG [Thread-2034 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:18:07,011 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:18:07,011 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:18:07,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2] 2024-11-14T06:18:07,011 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47612, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:18:07,011 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:18:07,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:18:07,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:18:07,012 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:18:07,012 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:18:07,012 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:18:07,012 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47628, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:18:07,013 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47646, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:18:07,013 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47650, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:18:07,013 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T06:18:07,014 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47634, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:18:07,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2] 2024-11-14T06:18:07,014 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47652, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:18:07,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-14T06:18:07,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2] 2024-11-14T06:18:07,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2] 2024-11-14T06:18:07,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T06:18:07,016 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T06:18:07,016 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2] 2024-11-14T06:18:07,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2] 2024-11-14T06:18:07,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2] 2024-11-14T06:18:07,018 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2] 2024-11-14T06:18:07,018 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T06:18:07,018 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T06:18:07,019 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2] 2024-11-14T06:18:07,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2] 2024-11-14T06:18:07,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on c151951bebfcfd871997dfedda0e3149 2024-11-14T06:18:07,026 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c151951bebfcfd871997dfedda0e3149 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T06:18:07,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=A 2024-11-14T06:18:07,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:07,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=B 2024-11-14T06:18:07,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:07,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=C 2024-11-14T06:18:07,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:07,047 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/55d4550cc1f44f80b21e9b4d3ec95ff5 is 50, key is test_row_0/A:col10/1731565087024/Put/seqid=0 2024-11-14T06:18:07,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742278_1454 (size=14341) 2024-11-14T06:18:07,059 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731565087048, max timestamp=1731565087056Num uniques 859; 2024-11-14T06:18:07,059 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731565087048, max timestamp=1731565087056Num uniques 859; 2024-11-14T06:18:07,059 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T06:18:07,059 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731565087048, max timestamp=1731565087056Num uniques 859; 2024-11-14T06:18:07,059 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T06:18:07,059 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T06:18:07,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:07,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47628 deadline: 1731565147074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:07,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:07,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:07,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:07,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:07,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47634 deadline: 1731565147079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:07,086 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:07,086 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:07,086 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:07,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:07,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47646 deadline: 1731565147080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:07,091 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:07,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:07,091 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:07,091 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:07,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731565147081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:07,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:07,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:07,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:07,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:07,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47606 deadline: 1731565147082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:07,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:07,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:07,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:07,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T06:18:07,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-14T06:18:07,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:07,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. as already flushing 2024-11-14T06:18:07,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:07,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:07,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:07,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:07,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:07,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47628 deadline: 1731565147187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:07,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:07,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:07,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47634 deadline: 1731565147187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:07,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:07,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:07,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:07,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:07,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:07,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:07,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47646 deadline: 1731565147197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:07,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:07,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:07,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:07,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:07,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731565147198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:07,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:07,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:07,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47606 deadline: 1731565147198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:07,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:07,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:07,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:07,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:07,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:07,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-14T06:18:07,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:07,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. as already flushing 2024-11-14T06:18:07,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:07,325 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:07,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:07,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:07,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T06:18:07,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:07,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47628 deadline: 1731565147397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:07,404 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:07,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:07,404 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:07,404 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:07,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47634 deadline: 1731565147397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:07,404 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:07,404 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:07,404 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:07,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:07,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47646 deadline: 1731565147407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:07,414 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:07,414 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:07,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:07,414 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:07,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731565147407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:07,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:07,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:07,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:07,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:07,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47606 deadline: 1731565147417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:07,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:07,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:07,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:07,457 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/55d4550cc1f44f80b21e9b4d3ec95ff5 2024-11-14T06:18:07,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-14T06:18:07,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:07,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. as already flushing 2024-11-14T06:18:07,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:07,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:07,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:07,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/1e4629de011c4997985e7e9c6f949ded is 50, key is test_row_0/B:col10/1731565087024/Put/seqid=0 2024-11-14T06:18:07,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:07,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742279_1455 (size=12001) 2024-11-14T06:18:07,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-14T06:18:07,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:07,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. as already flushing 2024-11-14T06:18:07,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:07,630 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:07,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:07,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:07,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T06:18:07,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:07,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47628 deadline: 1731565147707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:07,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:07,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:07,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:07,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:07,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47634 deadline: 1731565147717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:07,724 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:07,724 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:07,724 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:07,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:07,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47646 deadline: 1731565147727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:07,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:07,735 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:07,735 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:07,735 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:07,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731565147727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:07,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:07,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:07,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:07,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:07,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47606 deadline: 1731565147737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:07,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:07,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:07,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:07,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-14T06:18:07,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:07,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. as already flushing 2024-11-14T06:18:07,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:07,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:07,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:07,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:07,883 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/1e4629de011c4997985e7e9c6f949ded 2024-11-14T06:18:07,909 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/bec89b3e6de94c6f80f296728adea529 is 50, key is test_row_0/C:col10/1731565087024/Put/seqid=0 2024-11-14T06:18:07,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742280_1456 (size=12001) 2024-11-14T06:18:07,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-14T06:18:07,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:07,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. as already flushing 2024-11-14T06:18:07,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:07,935 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:07,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:07,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:08,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-14T06:18:08,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:08,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. as already flushing 2024-11-14T06:18:08,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:08,088 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:08,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:08,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:08,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T06:18:08,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:08,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47628 deadline: 1731565148227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:08,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:08,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:08,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:08,240 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-14T06:18:08,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:08,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. as already flushing 2024-11-14T06:18:08,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:08,240 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:08,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:08,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:08,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:08,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47634 deadline: 1731565148237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:08,247 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:08,247 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:08,247 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:08,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:08,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731565148248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:08,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:08,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:08,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:08,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:08,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47646 deadline: 1731565148248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:08,254 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:08,254 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:08,254 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:08,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:08,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47606 deadline: 1731565148257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:08,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:08,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:08,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:08,316 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/bec89b3e6de94c6f80f296728adea529 2024-11-14T06:18:08,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/55d4550cc1f44f80b21e9b4d3ec95ff5 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/55d4550cc1f44f80b21e9b4d3ec95ff5 2024-11-14T06:18:08,323 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/55d4550cc1f44f80b21e9b4d3ec95ff5, entries=200, sequenceid=11, filesize=14.0 K 2024-11-14T06:18:08,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/1e4629de011c4997985e7e9c6f949ded as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/1e4629de011c4997985e7e9c6f949ded 2024-11-14T06:18:08,327 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/1e4629de011c4997985e7e9c6f949ded, entries=150, sequenceid=11, filesize=11.7 K 2024-11-14T06:18:08,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/bec89b3e6de94c6f80f296728adea529 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/bec89b3e6de94c6f80f296728adea529 2024-11-14T06:18:08,331 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/bec89b3e6de94c6f80f296728adea529, entries=150, sequenceid=11, filesize=11.7 K 2024-11-14T06:18:08,332 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for c151951bebfcfd871997dfedda0e3149 in 1307ms, sequenceid=11, compaction requested=false 2024-11-14T06:18:08,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c151951bebfcfd871997dfedda0e3149: 2024-11-14T06:18:08,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-14T06:18:08,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:08,393 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing c151951bebfcfd871997dfedda0e3149 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-14T06:18:08,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=A 2024-11-14T06:18:08,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:18:08,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=B 2024-11-14T06:18:08,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:18:08,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=C 2024-11-14T06:18:08,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:18:08,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/7c98c295bb254185980907b085237c01 is 50, key is test_row_0/A:col10/1731565087079/Put/seqid=0 2024-11-14T06:18:08,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742281_1457 (size=12001) 2024-11-14T06:18:08,402 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/7c98c295bb254185980907b085237c01 2024-11-14T06:18:08,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/bcd52a32bc0f4dcc9447b9594c19f5af is 50, key is test_row_0/B:col10/1731565087079/Put/seqid=0 2024-11-14T06:18:08,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742282_1458 (size=12001) 2024-11-14T06:18:08,833 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/bcd52a32bc0f4dcc9447b9594c19f5af 2024-11-14T06:18:08,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/b17fd7006793483c8314f1dcec1a0332 is 50, key is test_row_0/C:col10/1731565087079/Put/seqid=0 2024-11-14T06:18:08,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742283_1459 (size=12001) 2024-11-14T06:18:09,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T06:18:09,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on c151951bebfcfd871997dfedda0e3149 2024-11-14T06:18:09,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. as already flushing 2024-11-14T06:18:09,266 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/b17fd7006793483c8314f1dcec1a0332 2024-11-14T06:18:09,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/7c98c295bb254185980907b085237c01 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/7c98c295bb254185980907b085237c01 2024-11-14T06:18:09,273 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/7c98c295bb254185980907b085237c01, entries=150, sequenceid=46, filesize=11.7 K 2024-11-14T06:18:09,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/bcd52a32bc0f4dcc9447b9594c19f5af as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/bcd52a32bc0f4dcc9447b9594c19f5af 2024-11-14T06:18:09,277 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/bcd52a32bc0f4dcc9447b9594c19f5af, entries=150, sequenceid=46, filesize=11.7 K 2024-11-14T06:18:09,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/b17fd7006793483c8314f1dcec1a0332 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/b17fd7006793483c8314f1dcec1a0332 2024-11-14T06:18:09,281 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/b17fd7006793483c8314f1dcec1a0332, entries=150, sequenceid=46, filesize=11.7 K 2024-11-14T06:18:09,282 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for c151951bebfcfd871997dfedda0e3149 in 889ms, sequenceid=46, compaction requested=false 2024-11-14T06:18:09,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for c151951bebfcfd871997dfedda0e3149: 2024-11-14T06:18:09,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:09,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-14T06:18:09,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-14T06:18:09,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on c151951bebfcfd871997dfedda0e3149 2024-11-14T06:18:09,284 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c151951bebfcfd871997dfedda0e3149 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-14T06:18:09,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=A 2024-11-14T06:18:09,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:09,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=B 2024-11-14T06:18:09,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:09,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=C 2024-11-14T06:18:09,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:09,285 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-14T06:18:09,285 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2650 sec 2024-11-14T06:18:09,287 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 2.2730 sec 2024-11-14T06:18:09,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/c645e8861fa9477382e22094daebe1ed is 50, key is test_row_0/A:col10/1731565089275/Put/seqid=0 2024-11-14T06:18:09,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742284_1460 (size=16681) 2024-11-14T06:18:09,322 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731565089306, max timestamp=1731565089320Num uniques 859; 2024-11-14T06:18:09,322 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-14T06:18:09,322 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731565089306, max timestamp=1731565089320Num uniques 859; 2024-11-14T06:18:09,323 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731565089306, max timestamp=1731565089320Num uniques 859; 2024-11-14T06:18:09,322 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-14T06:18:09,323 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-14T06:18:09,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:09,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47606 deadline: 1731565149380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:09,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:09,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:09,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:09,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:09,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47628 deadline: 1731565149391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:09,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:09,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:09,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:09,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:09,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47634 deadline: 1731565149392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:09,409 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:09,409 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:09,409 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:09,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:09,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731565149393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:09,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:09,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:09,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:09,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:09,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47646 deadline: 1731565149399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:09,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:09,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:09,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:09,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:09,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47606 deadline: 1731565149497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:09,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:09,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:09,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:09,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:09,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47628 deadline: 1731565149517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:09,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:09,523 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:09,523 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:09,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47634 deadline: 1731565149517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:09,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:09,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:09,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:09,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:09,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:09,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731565149517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:09,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:09,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:09,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:09,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:09,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47646 deadline: 1731565149517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:09,524 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:09,524 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:09,524 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:09,701 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/c645e8861fa9477382e22094daebe1ed 2024-11-14T06:18:09,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/e2fc54c3c3f241209f717f2f77e4641e is 50, key is test_row_0/B:col10/1731565089275/Put/seqid=0 2024-11-14T06:18:09,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:09,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47606 deadline: 1731565149718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:09,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:09,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:09,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:09,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742285_1461 (size=12001) 2024-11-14T06:18:09,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:09,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47628 deadline: 1731565149727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:09,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:09,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47634 deadline: 1731565149727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:09,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:09,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:09,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:09,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:09,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:09,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:09,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731565149727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:09,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:09,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:09,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:09,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:09,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47646 deadline: 1731565149728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:09,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:09,742 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:09,742 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:09,742 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:09,972 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T06:18:10,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:10,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47606 deadline: 1731565150027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:10,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:10,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:10,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:10,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:10,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47628 deadline: 1731565150047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:10,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:10,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:10,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:10,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47634 deadline: 1731565150047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:10,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:10,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:10,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:10,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:10,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:10,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731565150048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:10,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:10,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:10,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47646 deadline: 1731565150048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:10,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:10,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:10,060 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:10,060 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:10,060 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:10,125 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/e2fc54c3c3f241209f717f2f77e4641e 2024-11-14T06:18:10,132 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/6676e93a2d1045cfa2e4b18c305b10ea is 50, key is test_row_0/C:col10/1731565089275/Put/seqid=0 2024-11-14T06:18:10,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742286_1462 (size=12001) 2024-11-14T06:18:10,536 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/6676e93a2d1045cfa2e4b18c305b10ea 2024-11-14T06:18:10,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/c645e8861fa9477382e22094daebe1ed as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/c645e8861fa9477382e22094daebe1ed 2024-11-14T06:18:10,543 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/c645e8861fa9477382e22094daebe1ed, entries=250, sequenceid=57, filesize=16.3 K 2024-11-14T06:18:10,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/e2fc54c3c3f241209f717f2f77e4641e as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/e2fc54c3c3f241209f717f2f77e4641e 2024-11-14T06:18:10,546 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/e2fc54c3c3f241209f717f2f77e4641e, entries=150, sequenceid=57, filesize=11.7 K 2024-11-14T06:18:10,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/6676e93a2d1045cfa2e4b18c305b10ea as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/6676e93a2d1045cfa2e4b18c305b10ea 2024-11-14T06:18:10,556 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/6676e93a2d1045cfa2e4b18c305b10ea, entries=150, sequenceid=57, filesize=11.7 K 2024-11-14T06:18:10,557 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=147.60 KB/151140 for c151951bebfcfd871997dfedda0e3149 in 1273ms, sequenceid=57, compaction requested=true 2024-11-14T06:18:10,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c151951bebfcfd871997dfedda0e3149: 2024-11-14T06:18:10,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c151951bebfcfd871997dfedda0e3149:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T06:18:10,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:10,558 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:18:10,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c151951bebfcfd871997dfedda0e3149:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T06:18:10,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:10,558 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:18:10,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c151951bebfcfd871997dfedda0e3149:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T06:18:10,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:18:10,558 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:18:10,558 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:18:10,559 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): c151951bebfcfd871997dfedda0e3149/A is initiating minor compaction (all files) 2024-11-14T06:18:10,559 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): c151951bebfcfd871997dfedda0e3149/B is initiating minor compaction (all files) 2024-11-14T06:18:10,559 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c151951bebfcfd871997dfedda0e3149/B in TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:10,559 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c151951bebfcfd871997dfedda0e3149/A in TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:10,559 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/1e4629de011c4997985e7e9c6f949ded, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/bcd52a32bc0f4dcc9447b9594c19f5af, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/e2fc54c3c3f241209f717f2f77e4641e] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp, totalSize=35.2 K 2024-11-14T06:18:10,559 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/55d4550cc1f44f80b21e9b4d3ec95ff5, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/7c98c295bb254185980907b085237c01, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/c645e8861fa9477382e22094daebe1ed] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp, totalSize=42.0 K 2024-11-14T06:18:10,559 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e4629de011c4997985e7e9c6f949ded, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731565087015 2024-11-14T06:18:10,559 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55d4550cc1f44f80b21e9b4d3ec95ff5, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731565087015 2024-11-14T06:18:10,559 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting bcd52a32bc0f4dcc9447b9594c19f5af, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731565087078 2024-11-14T06:18:10,559 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c98c295bb254185980907b085237c01, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731565087078 2024-11-14T06:18:10,559 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting e2fc54c3c3f241209f717f2f77e4641e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731565089275 2024-11-14T06:18:10,559 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting c645e8861fa9477382e22094daebe1ed, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731565089269 2024-11-14T06:18:10,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on c151951bebfcfd871997dfedda0e3149 2024-11-14T06:18:10,563 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c151951bebfcfd871997dfedda0e3149 3/3 column families, dataSize=154.31 KB heapSize=426 KB 2024-11-14T06:18:10,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=A 2024-11-14T06:18:10,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:18:10,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=B 2024-11-14T06:18:10,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:18:10,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=C 2024-11-14T06:18:10,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:18:10,569 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c151951bebfcfd871997dfedda0e3149#A#compaction#387 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:18:10,569 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/07e0dcf2b6554228a15be359ac41a892 is 50, key is test_row_0/A:col10/1731565089275/Put/seqid=0 2024-11-14T06:18:10,570 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c151951bebfcfd871997dfedda0e3149#B#compaction#388 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:18:10,571 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/2244ceab619c4cbfa02adbe698896887 is 50, key is test_row_0/B:col10/1731565089275/Put/seqid=0 2024-11-14T06:18:10,576 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/3fdf416388f048fd96526ddead040844 is 50, key is test_row_0/A:col10/1731565090562/Put/seqid=0 2024-11-14T06:18:10,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742288_1464 (size=12104) 2024-11-14T06:18:10,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:10,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47646 deadline: 1731565150568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:10,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742287_1463 (size=12104) 2024-11-14T06:18:10,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:10,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47634 deadline: 1731565150572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:10,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:10,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:10,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:10,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:10,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:10,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:10,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742289_1465 (size=14341) 2024-11-14T06:18:10,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:10,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47606 deadline: 1731565150574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:10,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:10,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:10,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731565150589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:10,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:10,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:10,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:10,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:10,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:10,598 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/07e0dcf2b6554228a15be359ac41a892 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/07e0dcf2b6554228a15be359ac41a892 2024-11-14T06:18:10,604 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c151951bebfcfd871997dfedda0e3149/A of c151951bebfcfd871997dfedda0e3149 into 07e0dcf2b6554228a15be359ac41a892(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:18:10,604 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c151951bebfcfd871997dfedda0e3149: 2024-11-14T06:18:10,604 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., storeName=c151951bebfcfd871997dfedda0e3149/A, priority=13, startTime=1731565090558; duration=0sec 2024-11-14T06:18:10,605 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:18:10,605 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c151951bebfcfd871997dfedda0e3149:A 2024-11-14T06:18:10,605 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:18:10,606 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:18:10,606 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): c151951bebfcfd871997dfedda0e3149/C is initiating minor compaction (all files) 2024-11-14T06:18:10,606 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c151951bebfcfd871997dfedda0e3149/C in TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:10,606 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/bec89b3e6de94c6f80f296728adea529, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/b17fd7006793483c8314f1dcec1a0332, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/6676e93a2d1045cfa2e4b18c305b10ea] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp, totalSize=35.2 K 2024-11-14T06:18:10,607 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting bec89b3e6de94c6f80f296728adea529, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731565087015 2024-11-14T06:18:10,607 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting b17fd7006793483c8314f1dcec1a0332, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731565087078 2024-11-14T06:18:10,607 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6676e93a2d1045cfa2e4b18c305b10ea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731565089275 2024-11-14T06:18:10,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:10,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47628 deadline: 1731565150600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:10,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:10,612 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:10,612 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:10,620 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c151951bebfcfd871997dfedda0e3149#C#compaction#390 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:18:10,620 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/e267e2a77d7b4879be77129380c4c09f is 50, key is test_row_0/C:col10/1731565089275/Put/seqid=0 2024-11-14T06:18:10,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742290_1466 (size=12104) 2024-11-14T06:18:10,631 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/e267e2a77d7b4879be77129380c4c09f as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/e267e2a77d7b4879be77129380c4c09f 2024-11-14T06:18:10,638 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c151951bebfcfd871997dfedda0e3149/C of c151951bebfcfd871997dfedda0e3149 into e267e2a77d7b4879be77129380c4c09f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:18:10,638 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c151951bebfcfd871997dfedda0e3149: 2024-11-14T06:18:10,638 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., storeName=c151951bebfcfd871997dfedda0e3149/C, priority=13, startTime=1731565090558; duration=0sec 2024-11-14T06:18:10,638 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:10,638 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c151951bebfcfd871997dfedda0e3149:C 2024-11-14T06:18:10,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:10,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47634 deadline: 1731565150698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:10,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:10,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47606 deadline: 1731565150698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:10,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:10,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:10,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:10,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:10,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:10,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:10,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:10,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731565150707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:10,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:10,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:10,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:10,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:10,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47628 deadline: 1731565150720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:10,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:10,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:10,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:10,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:10,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47606 deadline: 1731565150917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:10,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:10,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:10,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47634 deadline: 1731565150917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:10,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:10,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:10,926 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:10,926 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:10,926 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:10,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:10,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731565150927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:10,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:10,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:10,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47628 deadline: 1731565150927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:10,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:10,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:10,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:10,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:10,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:10,981 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/2244ceab619c4cbfa02adbe698896887 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/2244ceab619c4cbfa02adbe698896887 2024-11-14T06:18:10,984 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c151951bebfcfd871997dfedda0e3149/B of c151951bebfcfd871997dfedda0e3149 into 2244ceab619c4cbfa02adbe698896887(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:18:10,984 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c151951bebfcfd871997dfedda0e3149: 2024-11-14T06:18:10,984 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., storeName=c151951bebfcfd871997dfedda0e3149/B, priority=13, startTime=1731565090558; duration=0sec 2024-11-14T06:18:10,984 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:10,984 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c151951bebfcfd871997dfedda0e3149:B 2024-11-14T06:18:10,994 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/3fdf416388f048fd96526ddead040844 2024-11-14T06:18:11,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/73384b33e24e487bb7be804ad03a80ce is 50, key is test_row_0/B:col10/1731565090562/Put/seqid=0 2024-11-14T06:18:11,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742291_1467 (size=12001) 2024-11-14T06:18:11,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T06:18:11,178 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:18:11,179 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T06:18:11,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-14T06:18:11,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-14T06:18:11,181 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T06:18:11,182 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T06:18:11,182 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T06:18:11,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:11,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47606 deadline: 1731565151237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:11,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:11,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47634 deadline: 1731565151237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:11,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:11,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:11,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:11,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:11,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:11,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:11,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:11,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731565151247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:11,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:11,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:11,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47628 deadline: 1731565151247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:11,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:11,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:11,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:11,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:11,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:11,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-14T06:18:11,338 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-14T06:18:11,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:11,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. as already flushing 2024-11-14T06:18:11,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:11,338 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:11,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:11,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:11,410 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/73384b33e24e487bb7be804ad03a80ce 2024-11-14T06:18:11,416 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/cb3301a633ff4e4bb18c27e73529bbd1 is 50, key is test_row_0/C:col10/1731565090562/Put/seqid=0 2024-11-14T06:18:11,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742292_1468 (size=12001) 2024-11-14T06:18:11,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-14T06:18:11,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:11,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. as already flushing 2024-11-14T06:18:11,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:11,491 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:11,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:11,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:11,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-14T06:18:11,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:11,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47646 deadline: 1731565151607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:11,612 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to ef93ef3a83c5,42283,1731564965945 for 'test_row_1' in c151951bebfcfd871997dfedda0e3149 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T06:18:11,614 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:11,614 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:11,614 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:11,643 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-14T06:18:11,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:11,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. as already flushing 2024-11-14T06:18:11,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:11,643 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:11,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:11,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:11,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:11,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47606 deadline: 1731565151747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:11,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:11,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:11,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47634 deadline: 1731565151747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:11,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:11,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:11,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:11,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:11,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:11,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:11,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47628 deadline: 1731565151758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:11,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:11,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:11,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:11,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:11,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731565151767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:11,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:11,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:11,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:11,798 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-14T06:18:11,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:11,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. as already flushing 2024-11-14T06:18:11,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:11,799 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:11,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:11,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:11,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-14T06:18:11,819 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/cb3301a633ff4e4bb18c27e73529bbd1 2024-11-14T06:18:11,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/3fdf416388f048fd96526ddead040844 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/3fdf416388f048fd96526ddead040844 2024-11-14T06:18:11,828 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/3fdf416388f048fd96526ddead040844, entries=200, sequenceid=93, filesize=14.0 K 2024-11-14T06:18:11,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/73384b33e24e487bb7be804ad03a80ce as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/73384b33e24e487bb7be804ad03a80ce 2024-11-14T06:18:11,831 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/73384b33e24e487bb7be804ad03a80ce, entries=150, sequenceid=93, filesize=11.7 K 2024-11-14T06:18:11,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/cb3301a633ff4e4bb18c27e73529bbd1 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/cb3301a633ff4e4bb18c27e73529bbd1 2024-11-14T06:18:11,835 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/cb3301a633ff4e4bb18c27e73529bbd1, entries=150, sequenceid=93, filesize=11.7 K 2024-11-14T06:18:11,836 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~425.95 KB/436176, currentSize=33.54 KB/34350 for c151951bebfcfd871997dfedda0e3149 in 1272ms, sequenceid=93, compaction requested=false 2024-11-14T06:18:11,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c151951bebfcfd871997dfedda0e3149: 2024-11-14T06:18:11,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-14T06:18:11,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:11,952 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing c151951bebfcfd871997dfedda0e3149 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-14T06:18:11,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=A 2024-11-14T06:18:11,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:11,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=B 2024-11-14T06:18:11,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:11,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=C 2024-11-14T06:18:11,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:11,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/5cbd618ef7804b8b90894e7009100d91 is 50, key is test_row_1/A:col10/1731565090579/Put/seqid=0 2024-11-14T06:18:11,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742293_1469 (size=9657) 2024-11-14T06:18:12,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-14T06:18:12,377 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/5cbd618ef7804b8b90894e7009100d91 2024-11-14T06:18:12,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/ddeecb0e35784d3ca154ae5407f43836 is 50, key is test_row_1/B:col10/1731565090579/Put/seqid=0 2024-11-14T06:18:12,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742294_1470 (size=9657) 2024-11-14T06:18:12,433 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/ddeecb0e35784d3ca154ae5407f43836 2024-11-14T06:18:12,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/5e55c53998d047ef9c974103fa07108a is 50, key is test_row_1/C:col10/1731565090579/Put/seqid=0 2024-11-14T06:18:12,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742295_1471 (size=9657) 2024-11-14T06:18:12,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. as already flushing 2024-11-14T06:18:12,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on c151951bebfcfd871997dfedda0e3149 2024-11-14T06:18:12,814 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731565092797, max timestamp=1731565092810Num uniques 859; 2024-11-14T06:18:12,814 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T06:18:12,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731565092797, max timestamp=1731565092810Num uniques 859; 2024-11-14T06:18:12,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T06:18:12,817 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731565092797, max timestamp=1731565092810Num uniques 859; 2024-11-14T06:18:12,817 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T06:18:12,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:12,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47606 deadline: 1731565152850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:12,858 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731565092833, max timestamp=1731565092854Num uniques 1068; 2024-11-14T06:18:12,858 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731565092833, max timestamp=1731565092854Num uniques 1068; 2024-11-14T06:18:12,858 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-11-14T06:18:12,858 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-11-14T06:18:12,858 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731565092833, max timestamp=1731565092854Num uniques 1068; 2024-11-14T06:18:12,858 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-11-14T06:18:12,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:12,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:12,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:12,861 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/5e55c53998d047ef9c974103fa07108a 2024-11-14T06:18:12,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/5cbd618ef7804b8b90894e7009100d91 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/5cbd618ef7804b8b90894e7009100d91 2024-11-14T06:18:12,868 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/5cbd618ef7804b8b90894e7009100d91, entries=100, sequenceid=104, filesize=9.4 K 2024-11-14T06:18:12,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/ddeecb0e35784d3ca154ae5407f43836 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/ddeecb0e35784d3ca154ae5407f43836 2024-11-14T06:18:12,873 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/ddeecb0e35784d3ca154ae5407f43836, entries=100, sequenceid=104, filesize=9.4 K 2024-11-14T06:18:12,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/5e55c53998d047ef9c974103fa07108a as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/5e55c53998d047ef9c974103fa07108a 2024-11-14T06:18:12,881 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/5e55c53998d047ef9c974103fa07108a, entries=100, sequenceid=104, filesize=9.4 K 2024-11-14T06:18:12,882 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=73.80 KB/75570 for c151951bebfcfd871997dfedda0e3149 in 930ms, sequenceid=104, compaction requested=true 2024-11-14T06:18:12,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for c151951bebfcfd871997dfedda0e3149: 2024-11-14T06:18:12,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:12,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-14T06:18:12,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on c151951bebfcfd871997dfedda0e3149 2024-11-14T06:18:12,882 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c151951bebfcfd871997dfedda0e3149 3/3 column families, dataSize=80.51 KB heapSize=215.39 KB 2024-11-14T06:18:12,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-14T06:18:12,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=A 2024-11-14T06:18:12,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:18:12,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=B 2024-11-14T06:18:12,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:18:12,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=C 2024-11-14T06:18:12,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:18:12,885 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-14T06:18:12,885 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7010 sec 2024-11-14T06:18:12,887 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/de4d79515e584fc4a70c3465d9559296 is 50, key is test_row_0/A:col10/1731565092870/Put/seqid=0 2024-11-14T06:18:12,890 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.7090 sec 2024-11-14T06:18:12,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742296_1472 (size=19321) 2024-11-14T06:18:12,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:12,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731565152944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:12,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:12,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:12,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47634 deadline: 1731565152945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:12,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:12,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:12,951 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:12,951 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:12,951 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:12,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:12,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47628 deadline: 1731565152948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:12,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:12,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:12,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:12,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:12,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47606 deadline: 1731565152968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:12,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:12,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:12,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:13,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:13,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731565153057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:13,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:13,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:13,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:13,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47634 deadline: 1731565153057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:13,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:13,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:13,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:13,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:13,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:13,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47628 deadline: 1731565153067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:13,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:13,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:13,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:13,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:13,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47606 deadline: 1731565153187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:13,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:13,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:13,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:13,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:13,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731565153268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:13,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:13,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:13,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:13,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:13,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47634 deadline: 1731565153277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:13,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:13,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:13,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:13,299 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/de4d79515e584fc4a70c3465d9559296 2024-11-14T06:18:13,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:13,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47628 deadline: 1731565153287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:13,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:13,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:13,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:13,305 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/0ae6c5d273964df693dcdf20120fe48c is 50, key is test_row_0/B:col10/1731565092870/Put/seqid=0 2024-11-14T06:18:13,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742297_1473 (size=12151) 2024-11-14T06:18:13,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-14T06:18:13,328 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:18:13,328 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T06:18:13,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-14T06:18:13,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-14T06:18:13,330 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T06:18:13,330 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T06:18:13,330 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T06:18:13,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-14T06:18:13,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-14T06:18:13,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:13,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. as already flushing 2024-11-14T06:18:13,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:13,482 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:13,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:13,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:13,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:13,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47606 deadline: 1731565153507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:13,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:13,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:13,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:13,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:13,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731565153587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:13,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:13,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:13,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:13,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:13,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47634 deadline: 1731565153597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:13,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:13,606 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:13,606 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:13,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:13,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47628 deadline: 1731565153607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:13,617 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:13,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:13,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:13,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:13,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47646 deadline: 1731565153628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:13,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-14T06:18:13,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:13,635 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to ef93ef3a83c5,42283,1731564965945 for 'test_row_1' in c151951bebfcfd871997dfedda0e3149 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4236 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T06:18:13,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. as already flushing 2024-11-14T06:18:13,635 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:13,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:13,635 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:13,635 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:13,635 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:13,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:13,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:13,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-14T06:18:13,710 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/0ae6c5d273964df693dcdf20120fe48c 2024-11-14T06:18:13,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/576916c40e214785b18324a8bd3c28ac is 50, key is test_row_0/C:col10/1731565092870/Put/seqid=0 2024-11-14T06:18:13,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742298_1474 (size=12151) 2024-11-14T06:18:13,730 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/576916c40e214785b18324a8bd3c28ac 2024-11-14T06:18:13,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/de4d79515e584fc4a70c3465d9559296 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/de4d79515e584fc4a70c3465d9559296 2024-11-14T06:18:13,740 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/de4d79515e584fc4a70c3465d9559296, entries=300, sequenceid=148, filesize=18.9 K 2024-11-14T06:18:13,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/0ae6c5d273964df693dcdf20120fe48c as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/0ae6c5d273964df693dcdf20120fe48c 2024-11-14T06:18:13,743 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/0ae6c5d273964df693dcdf20120fe48c, entries=150, sequenceid=148, filesize=11.9 K 2024-11-14T06:18:13,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/576916c40e214785b18324a8bd3c28ac as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/576916c40e214785b18324a8bd3c28ac 2024-11-14T06:18:13,747 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/576916c40e214785b18324a8bd3c28ac, entries=150, sequenceid=148, filesize=11.9 K 2024-11-14T06:18:13,748 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~215.34 KB/220512, currentSize=100.63 KB/103050 for c151951bebfcfd871997dfedda0e3149 in 866ms, sequenceid=148, compaction requested=true 2024-11-14T06:18:13,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c151951bebfcfd871997dfedda0e3149: 2024-11-14T06:18:13,748 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T06:18:13,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c151951bebfcfd871997dfedda0e3149:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T06:18:13,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:13,748 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T06:18:13,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c151951bebfcfd871997dfedda0e3149:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T06:18:13,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:13,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c151951bebfcfd871997dfedda0e3149:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T06:18:13,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:18:13,749 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45913 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T06:18:13,749 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): c151951bebfcfd871997dfedda0e3149/B is initiating minor compaction (all files) 2024-11-14T06:18:13,749 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c151951bebfcfd871997dfedda0e3149/B in TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:13,750 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/2244ceab619c4cbfa02adbe698896887, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/73384b33e24e487bb7be804ad03a80ce, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/ddeecb0e35784d3ca154ae5407f43836, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/0ae6c5d273964df693dcdf20120fe48c] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp, totalSize=44.8 K 2024-11-14T06:18:13,750 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55423 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T06:18:13,750 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): c151951bebfcfd871997dfedda0e3149/A is initiating minor compaction (all files) 2024-11-14T06:18:13,750 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c151951bebfcfd871997dfedda0e3149/A in TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:13,750 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/07e0dcf2b6554228a15be359ac41a892, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/3fdf416388f048fd96526ddead040844, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/5cbd618ef7804b8b90894e7009100d91, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/de4d79515e584fc4a70c3465d9559296] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp, totalSize=54.1 K 2024-11-14T06:18:13,750 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 2244ceab619c4cbfa02adbe698896887, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731565089275 2024-11-14T06:18:13,750 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07e0dcf2b6554228a15be359ac41a892, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731565089275 2024-11-14T06:18:13,750 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 73384b33e24e487bb7be804ad03a80ce, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731565089390 2024-11-14T06:18:13,750 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3fdf416388f048fd96526ddead040844, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731565089363 2024-11-14T06:18:13,750 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting ddeecb0e35784d3ca154ae5407f43836, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731565090578 2024-11-14T06:18:13,750 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5cbd618ef7804b8b90894e7009100d91, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731565090578 2024-11-14T06:18:13,751 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ae6c5d273964df693dcdf20120fe48c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731565092870 2024-11-14T06:18:13,751 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting de4d79515e584fc4a70c3465d9559296, keycount=300, bloomtype=ROW, size=18.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731565092863 2024-11-14T06:18:13,758 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c151951bebfcfd871997dfedda0e3149#B#compaction#400 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:18:13,759 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c151951bebfcfd871997dfedda0e3149#A#compaction#399 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:18:13,759 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/6c50e7d4fc85433b854cf578457cb0fc is 50, key is test_row_0/B:col10/1731565092870/Put/seqid=0 2024-11-14T06:18:13,759 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/70b723c1355f440b8e7de2269758d7f6 is 50, key is test_row_0/A:col10/1731565092870/Put/seqid=0 2024-11-14T06:18:13,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742300_1476 (size=12391) 2024-11-14T06:18:13,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742299_1475 (size=12391) 2024-11-14T06:18:13,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-14T06:18:13,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:13,788 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing c151951bebfcfd871997dfedda0e3149 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-14T06:18:13,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=A 2024-11-14T06:18:13,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:13,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=B 2024-11-14T06:18:13,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:13,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=C 2024-11-14T06:18:13,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:13,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/a725821d5a434587932956d29d135c86 is 50, key is test_row_0/A:col10/1731565092937/Put/seqid=0 2024-11-14T06:18:13,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742301_1477 (size=12151) 2024-11-14T06:18:13,799 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/a725821d5a434587932956d29d135c86 2024-11-14T06:18:13,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/079a007e90404f65bd7a46b5742719ed is 50, key is test_row_0/B:col10/1731565092937/Put/seqid=0 2024-11-14T06:18:13,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742302_1478 (size=12151) 2024-11-14T06:18:13,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-14T06:18:14,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on c151951bebfcfd871997dfedda0e3149 2024-11-14T06:18:14,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. as already flushing 2024-11-14T06:18:14,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:14,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47634 deadline: 1731565154117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:14,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:14,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47606 deadline: 1731565154119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:14,127 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:14,127 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:14,127 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:14,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:14,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:14,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:14,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:14,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731565154120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:14,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:14,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:14,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:14,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:14,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47628 deadline: 1731565154128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:14,135 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:14,135 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:14,135 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:14,172 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/6c50e7d4fc85433b854cf578457cb0fc as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/6c50e7d4fc85433b854cf578457cb0fc 2024-11-14T06:18:14,177 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c151951bebfcfd871997dfedda0e3149/B of c151951bebfcfd871997dfedda0e3149 into 6c50e7d4fc85433b854cf578457cb0fc(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:18:14,177 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c151951bebfcfd871997dfedda0e3149: 2024-11-14T06:18:14,177 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., storeName=c151951bebfcfd871997dfedda0e3149/B, priority=12, startTime=1731565093748; duration=0sec 2024-11-14T06:18:14,177 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:18:14,177 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c151951bebfcfd871997dfedda0e3149:B 2024-11-14T06:18:14,178 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T06:18:14,179 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45913 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T06:18:14,179 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): c151951bebfcfd871997dfedda0e3149/C is initiating minor compaction (all files) 2024-11-14T06:18:14,179 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c151951bebfcfd871997dfedda0e3149/C in TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:14,179 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/e267e2a77d7b4879be77129380c4c09f, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/cb3301a633ff4e4bb18c27e73529bbd1, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/5e55c53998d047ef9c974103fa07108a, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/576916c40e214785b18324a8bd3c28ac] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp, totalSize=44.8 K 2024-11-14T06:18:14,179 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting e267e2a77d7b4879be77129380c4c09f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731565089275 2024-11-14T06:18:14,180 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting cb3301a633ff4e4bb18c27e73529bbd1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731565089390 2024-11-14T06:18:14,180 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e55c53998d047ef9c974103fa07108a, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731565090578 2024-11-14T06:18:14,180 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/70b723c1355f440b8e7de2269758d7f6 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/70b723c1355f440b8e7de2269758d7f6 2024-11-14T06:18:14,180 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 576916c40e214785b18324a8bd3c28ac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731565092870 2024-11-14T06:18:14,185 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c151951bebfcfd871997dfedda0e3149/A of c151951bebfcfd871997dfedda0e3149 into 70b723c1355f440b8e7de2269758d7f6(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:18:14,186 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c151951bebfcfd871997dfedda0e3149: 2024-11-14T06:18:14,186 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., storeName=c151951bebfcfd871997dfedda0e3149/A, priority=12, startTime=1731565093748; duration=0sec 2024-11-14T06:18:14,186 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:14,186 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c151951bebfcfd871997dfedda0e3149:A 2024-11-14T06:18:14,191 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c151951bebfcfd871997dfedda0e3149#C#compaction#403 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:18:14,191 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/bcc49c6cbc424263871d55cf751f57b6 is 50, key is test_row_0/C:col10/1731565092870/Put/seqid=0 2024-11-14T06:18:14,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742303_1479 (size=12391) 2024-11-14T06:18:14,204 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/bcc49c6cbc424263871d55cf751f57b6 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/bcc49c6cbc424263871d55cf751f57b6 2024-11-14T06:18:14,209 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c151951bebfcfd871997dfedda0e3149/C of c151951bebfcfd871997dfedda0e3149 into bcc49c6cbc424263871d55cf751f57b6(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:18:14,209 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c151951bebfcfd871997dfedda0e3149: 2024-11-14T06:18:14,209 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., storeName=c151951bebfcfd871997dfedda0e3149/C, priority=12, startTime=1731565093748; duration=0sec 2024-11-14T06:18:14,209 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:14,209 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c151951bebfcfd871997dfedda0e3149:C 2024-11-14T06:18:14,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:14,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47606 deadline: 1731565154237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:14,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:14,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731565154237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:14,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:14,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:14,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:14,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:14,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:14,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:14,283 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/079a007e90404f65bd7a46b5742719ed 2024-11-14T06:18:14,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/0df943bf6c5c4228a6d6aee2169ae0e2 is 50, key is test_row_0/C:col10/1731565092937/Put/seqid=0 2024-11-14T06:18:14,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742304_1480 (size=12151) 2024-11-14T06:18:14,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:14,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47606 deadline: 1731565154448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:14,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:14,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:14,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:14,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:14,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731565154457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:14,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:14,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:14,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:14,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-14T06:18:14,709 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/0df943bf6c5c4228a6d6aee2169ae0e2 2024-11-14T06:18:14,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/a725821d5a434587932956d29d135c86 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/a725821d5a434587932956d29d135c86 2024-11-14T06:18:14,722 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/a725821d5a434587932956d29d135c86, entries=150, sequenceid=166, filesize=11.9 K 2024-11-14T06:18:14,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/079a007e90404f65bd7a46b5742719ed as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/079a007e90404f65bd7a46b5742719ed 2024-11-14T06:18:14,729 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/079a007e90404f65bd7a46b5742719ed, entries=150, sequenceid=166, filesize=11.9 K 2024-11-14T06:18:14,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/0df943bf6c5c4228a6d6aee2169ae0e2 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/0df943bf6c5c4228a6d6aee2169ae0e2 2024-11-14T06:18:14,734 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/0df943bf6c5c4228a6d6aee2169ae0e2, entries=150, sequenceid=166, filesize=11.9 K 2024-11-14T06:18:14,735 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for c151951bebfcfd871997dfedda0e3149 in 946ms, sequenceid=166, compaction requested=false 2024-11-14T06:18:14,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for c151951bebfcfd871997dfedda0e3149: 2024-11-14T06:18:14,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:14,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-14T06:18:14,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-14T06:18:14,741 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-14T06:18:14,741 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4060 sec 2024-11-14T06:18:14,743 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.4140 sec 2024-11-14T06:18:14,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on c151951bebfcfd871997dfedda0e3149 2024-11-14T06:18:14,772 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c151951bebfcfd871997dfedda0e3149 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-14T06:18:14,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=A 2024-11-14T06:18:14,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:14,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=B 2024-11-14T06:18:14,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:14,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=C 2024-11-14T06:18:14,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:14,777 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/f2bc4974354f4c6caa26525d334df4ed is 50, key is test_row_0/A:col10/1731565094771/Put/seqid=0 2024-11-14T06:18:14,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742305_1481 (size=14541) 2024-11-14T06:18:14,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:14,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47606 deadline: 1731565154840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:14,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:14,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:14,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:14,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:14,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731565154853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:14,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:14,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:14,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:14,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:14,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47606 deadline: 1731565154957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:14,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:14,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:14,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:14,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:14,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731565154968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:14,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:14,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:14,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:15,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:15,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47628 deadline: 1731565155137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:15,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:15,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47634 deadline: 1731565155137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:15,146 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to ef93ef3a83c5,42283,1731564965945 for 'test_row_1' in c151951bebfcfd871997dfedda0e3149 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T06:18:15,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:15,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:15,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:15,147 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to ef93ef3a83c5,42283,1731564965945 for 'test_row_2' in c151951bebfcfd871997dfedda0e3149 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T06:18:15,147 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:15,147 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:15,147 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:15,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:15,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47606 deadline: 1731565155177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:15,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:15,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:15,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:15,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:15,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731565155187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:15,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:15,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:15,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:15,199 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/f2bc4974354f4c6caa26525d334df4ed 2024-11-14T06:18:15,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/9254b63b509a47b7b897cdbd2ac270e3 is 50, key is test_row_0/B:col10/1731565094771/Put/seqid=0 2024-11-14T06:18:15,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742306_1482 (size=12151) 2024-11-14T06:18:15,238 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/9254b63b509a47b7b897cdbd2ac270e3 2024-11-14T06:18:15,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/6a5ed1c0d6e74c819500767755b7dbf0 is 50, key is test_row_0/C:col10/1731565094771/Put/seqid=0 2024-11-14T06:18:15,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742307_1483 (size=12151) 2024-11-14T06:18:15,268 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/6a5ed1c0d6e74c819500767755b7dbf0 2024-11-14T06:18:15,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/f2bc4974354f4c6caa26525d334df4ed as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/f2bc4974354f4c6caa26525d334df4ed 2024-11-14T06:18:15,275 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/f2bc4974354f4c6caa26525d334df4ed, entries=200, sequenceid=184, filesize=14.2 K 2024-11-14T06:18:15,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/9254b63b509a47b7b897cdbd2ac270e3 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/9254b63b509a47b7b897cdbd2ac270e3 2024-11-14T06:18:15,279 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/9254b63b509a47b7b897cdbd2ac270e3, entries=150, sequenceid=184, filesize=11.9 K 2024-11-14T06:18:15,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/6a5ed1c0d6e74c819500767755b7dbf0 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/6a5ed1c0d6e74c819500767755b7dbf0 2024-11-14T06:18:15,283 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/6a5ed1c0d6e74c819500767755b7dbf0, entries=150, sequenceid=184, filesize=11.9 K 2024-11-14T06:18:15,283 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for c151951bebfcfd871997dfedda0e3149 in 511ms, sequenceid=184, compaction requested=true 2024-11-14T06:18:15,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c151951bebfcfd871997dfedda0e3149: 2024-11-14T06:18:15,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c151951bebfcfd871997dfedda0e3149:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T06:18:15,283 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:18:15,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:15,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c151951bebfcfd871997dfedda0e3149:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T06:18:15,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:15,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c151951bebfcfd871997dfedda0e3149:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T06:18:15,284 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:18:15,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:18:15,287 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39083 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:18:15,287 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:18:15,287 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): c151951bebfcfd871997dfedda0e3149/A is initiating minor compaction (all files) 2024-11-14T06:18:15,287 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): c151951bebfcfd871997dfedda0e3149/B is initiating minor compaction (all files) 2024-11-14T06:18:15,287 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c151951bebfcfd871997dfedda0e3149/A in TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:15,287 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c151951bebfcfd871997dfedda0e3149/B in TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:15,287 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/70b723c1355f440b8e7de2269758d7f6, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/a725821d5a434587932956d29d135c86, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/f2bc4974354f4c6caa26525d334df4ed] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp, totalSize=38.2 K 2024-11-14T06:18:15,287 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/6c50e7d4fc85433b854cf578457cb0fc, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/079a007e90404f65bd7a46b5742719ed, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/9254b63b509a47b7b897cdbd2ac270e3] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp, totalSize=35.8 K 2024-11-14T06:18:15,288 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c50e7d4fc85433b854cf578457cb0fc, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731565092870 2024-11-14T06:18:15,288 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70b723c1355f440b8e7de2269758d7f6, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731565092870 2024-11-14T06:18:15,289 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 079a007e90404f65bd7a46b5742719ed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731565092937 2024-11-14T06:18:15,289 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting a725821d5a434587932956d29d135c86, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731565092937 2024-11-14T06:18:15,289 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2bc4974354f4c6caa26525d334df4ed, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731565094049 2024-11-14T06:18:15,289 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 9254b63b509a47b7b897cdbd2ac270e3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731565094049 2024-11-14T06:18:15,296 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c151951bebfcfd871997dfedda0e3149#B#compaction#408 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:18:15,296 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/383b634c434f4d10bb2d6a2709ab7fa1 is 50, key is test_row_0/B:col10/1731565094771/Put/seqid=0 2024-11-14T06:18:15,297 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c151951bebfcfd871997dfedda0e3149#A#compaction#409 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:18:15,297 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/5b48a8b71e314300a7c7df412c365e43 is 50, key is test_row_0/A:col10/1731565094771/Put/seqid=0 2024-11-14T06:18:15,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742308_1484 (size=12493) 2024-11-14T06:18:15,308 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/383b634c434f4d10bb2d6a2709ab7fa1 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/383b634c434f4d10bb2d6a2709ab7fa1 2024-11-14T06:18:15,320 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c151951bebfcfd871997dfedda0e3149/B of c151951bebfcfd871997dfedda0e3149 into 383b634c434f4d10bb2d6a2709ab7fa1(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:18:15,320 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c151951bebfcfd871997dfedda0e3149: 2024-11-14T06:18:15,320 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., storeName=c151951bebfcfd871997dfedda0e3149/B, priority=13, startTime=1731565095283; duration=0sec 2024-11-14T06:18:15,320 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:18:15,320 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c151951bebfcfd871997dfedda0e3149:B 2024-11-14T06:18:15,320 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:18:15,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742309_1485 (size=12493) 2024-11-14T06:18:15,323 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:18:15,324 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): c151951bebfcfd871997dfedda0e3149/C is initiating minor compaction (all files) 2024-11-14T06:18:15,324 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c151951bebfcfd871997dfedda0e3149/C in TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:15,324 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/bcc49c6cbc424263871d55cf751f57b6, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/0df943bf6c5c4228a6d6aee2169ae0e2, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/6a5ed1c0d6e74c819500767755b7dbf0] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp, totalSize=35.8 K 2024-11-14T06:18:15,324 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting bcc49c6cbc424263871d55cf751f57b6, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731565092870 2024-11-14T06:18:15,324 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 0df943bf6c5c4228a6d6aee2169ae0e2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731565092937 2024-11-14T06:18:15,325 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a5ed1c0d6e74c819500767755b7dbf0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731565094049 2024-11-14T06:18:15,331 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c151951bebfcfd871997dfedda0e3149#C#compaction#410 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:18:15,331 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/d9a086ed6ef9415d8eb9f205550149a4 is 50, key is test_row_0/C:col10/1731565094771/Put/seqid=0 2024-11-14T06:18:15,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742310_1486 (size=12493) 2024-11-14T06:18:15,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-14T06:18:15,478 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:18:15,478 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T06:18:15,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-14T06:18:15,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-14T06:18:15,480 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T06:18:15,481 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T06:18:15,481 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T06:18:15,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on c151951bebfcfd871997dfedda0e3149 2024-11-14T06:18:15,502 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c151951bebfcfd871997dfedda0e3149 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-14T06:18:15,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=A 2024-11-14T06:18:15,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:15,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=B 2024-11-14T06:18:15,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:15,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=C 2024-11-14T06:18:15,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:15,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/07aff5d543a54275a83eb76f4b2378f6 is 50, key is test_row_0/A:col10/1731565095501/Put/seqid=0 2024-11-14T06:18:15,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742311_1487 (size=14541) 2024-11-14T06:18:15,538 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-14T06:18:15,538 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-14T06:18:15,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:15,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47606 deadline: 1731565155546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:15,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:15,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:15,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:15,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:15,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731565155556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:15,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:15,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:15,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:15,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-14T06:18:15,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-14T06:18:15,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:15,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. as already flushing 2024-11-14T06:18:15,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:15,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:15,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:15,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:15,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:15,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731565155667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:15,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:15,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:15,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:15,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47606 deadline: 1731565155667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:15,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:15,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:15,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:15,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:15,744 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/5b48a8b71e314300a7c7df412c365e43 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/5b48a8b71e314300a7c7df412c365e43 2024-11-14T06:18:15,748 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/d9a086ed6ef9415d8eb9f205550149a4 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/d9a086ed6ef9415d8eb9f205550149a4 2024-11-14T06:18:15,755 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c151951bebfcfd871997dfedda0e3149/C of c151951bebfcfd871997dfedda0e3149 into d9a086ed6ef9415d8eb9f205550149a4(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:18:15,755 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c151951bebfcfd871997dfedda0e3149/A of c151951bebfcfd871997dfedda0e3149 into 5b48a8b71e314300a7c7df412c365e43(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:18:15,755 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c151951bebfcfd871997dfedda0e3149: 2024-11-14T06:18:15,755 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c151951bebfcfd871997dfedda0e3149: 2024-11-14T06:18:15,755 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., storeName=c151951bebfcfd871997dfedda0e3149/C, priority=13, startTime=1731565095284; duration=0sec 2024-11-14T06:18:15,755 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., storeName=c151951bebfcfd871997dfedda0e3149/A, priority=13, startTime=1731565095283; duration=0sec 2024-11-14T06:18:15,755 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:15,755 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c151951bebfcfd871997dfedda0e3149:C 2024-11-14T06:18:15,755 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:15,755 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c151951bebfcfd871997dfedda0e3149:A 2024-11-14T06:18:15,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-14T06:18:15,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:15,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. as already flushing 2024-11-14T06:18:15,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:15,791 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:15,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:15,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:15,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-14T06:18:15,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:15,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47606 deadline: 1731565155877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:15,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:15,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:15,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:15,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:15,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731565155877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:15,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:15,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:15,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:15,919 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/07aff5d543a54275a83eb76f4b2378f6 2024-11-14T06:18:15,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/cbdf47e6151b4a488be97aa0873c2ec1 is 50, key is test_row_0/B:col10/1731565095501/Put/seqid=0 2024-11-14T06:18:15,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742312_1488 (size=12151) 2024-11-14T06:18:15,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-14T06:18:15,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:15,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. as already flushing 2024-11-14T06:18:15,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:15,945 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:15,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:15,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:16,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-14T06:18:16,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:16,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. as already flushing 2024-11-14T06:18:16,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:16,097 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:16,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:16,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:16,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-14T06:18:16,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:16,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47606 deadline: 1731565156187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:16,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:16,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:16,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:16,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:16,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731565156197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:16,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:16,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:16,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:16,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-14T06:18:16,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:16,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. as already flushing 2024-11-14T06:18:16,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:16,250 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:16,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:16,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:16,339 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/cbdf47e6151b4a488be97aa0873c2ec1 2024-11-14T06:18:16,348 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/0af4712533054a83b642ec6e8e3a6cb3 is 50, key is test_row_0/C:col10/1731565095501/Put/seqid=0 2024-11-14T06:18:16,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742313_1489 (size=12151) 2024-11-14T06:18:16,402 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-14T06:18:16,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:16,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. as already flushing 2024-11-14T06:18:16,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:16,402 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:16,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:16,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:16,558 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-14T06:18:16,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:16,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. as already flushing 2024-11-14T06:18:16,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:16,558 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:16,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:16,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:16,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-14T06:18:16,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:16,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47606 deadline: 1731565156697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:16,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:16,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:16,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:16,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-14T06:18:16,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:16,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. as already flushing 2024-11-14T06:18:16,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:16,711 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:16,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:16,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:16,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:16,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731565156717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:16,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:16,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:16,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:16,759 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/0af4712533054a83b642ec6e8e3a6cb3 2024-11-14T06:18:16,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/07aff5d543a54275a83eb76f4b2378f6 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/07aff5d543a54275a83eb76f4b2378f6 2024-11-14T06:18:16,778 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/07aff5d543a54275a83eb76f4b2378f6, entries=200, sequenceid=203, filesize=14.2 K 2024-11-14T06:18:16,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/cbdf47e6151b4a488be97aa0873c2ec1 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/cbdf47e6151b4a488be97aa0873c2ec1 2024-11-14T06:18:16,782 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/cbdf47e6151b4a488be97aa0873c2ec1, entries=150, sequenceid=203, filesize=11.9 K 2024-11-14T06:18:16,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/0af4712533054a83b642ec6e8e3a6cb3 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/0af4712533054a83b642ec6e8e3a6cb3 2024-11-14T06:18:16,789 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/0af4712533054a83b642ec6e8e3a6cb3, entries=150, sequenceid=203, filesize=11.9 K 2024-11-14T06:18:16,808 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for c151951bebfcfd871997dfedda0e3149 in 1306ms, sequenceid=203, compaction requested=false 2024-11-14T06:18:16,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c151951bebfcfd871997dfedda0e3149: 2024-11-14T06:18:16,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-14T06:18:16,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:16,863 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing c151951bebfcfd871997dfedda0e3149 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-14T06:18:16,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=A 2024-11-14T06:18:16,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:16,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=B 2024-11-14T06:18:16,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:16,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=C 2024-11-14T06:18:16,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:16,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/89262aa98ef54e3bbe607b97f7aac656 is 50, key is test_row_0/A:col10/1731565095554/Put/seqid=0 2024-11-14T06:18:16,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742314_1490 (size=12151) 2024-11-14T06:18:16,932 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/89262aa98ef54e3bbe607b97f7aac656 2024-11-14T06:18:16,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/77bd2c24b59e43cea71383ae2dacc68f is 50, key is test_row_0/B:col10/1731565095554/Put/seqid=0 2024-11-14T06:18:16,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742315_1491 (size=12151) 2024-11-14T06:18:16,980 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/77bd2c24b59e43cea71383ae2dacc68f 2024-11-14T06:18:16,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/be3f5aef906349cc8597b0e0c015880e is 50, key is test_row_0/C:col10/1731565095554/Put/seqid=0 2024-11-14T06:18:17,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742316_1492 (size=12151) 2024-11-14T06:18:17,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on c151951bebfcfd871997dfedda0e3149 2024-11-14T06:18:17,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. as already flushing 2024-11-14T06:18:17,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:17,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47634 deadline: 1731565157213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:17,222 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:17,222 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:17,222 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:17,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:17,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47628 deadline: 1731565157214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:17,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:17,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:17,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:17,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:17,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47634 deadline: 1731565157327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:17,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:17,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:17,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:17,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:17,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47628 deadline: 1731565157337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:17,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:17,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:17,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:17,408 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/be3f5aef906349cc8597b0e0c015880e 2024-11-14T06:18:17,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/89262aa98ef54e3bbe607b97f7aac656 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/89262aa98ef54e3bbe607b97f7aac656 2024-11-14T06:18:17,413 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/89262aa98ef54e3bbe607b97f7aac656, entries=150, sequenceid=219, filesize=11.9 K 2024-11-14T06:18:17,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/77bd2c24b59e43cea71383ae2dacc68f as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/77bd2c24b59e43cea71383ae2dacc68f 2024-11-14T06:18:17,418 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/77bd2c24b59e43cea71383ae2dacc68f, entries=150, sequenceid=219, filesize=11.9 K 2024-11-14T06:18:17,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/be3f5aef906349cc8597b0e0c015880e as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/be3f5aef906349cc8597b0e0c015880e 2024-11-14T06:18:17,422 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/be3f5aef906349cc8597b0e0c015880e, entries=150, sequenceid=219, filesize=11.9 K 2024-11-14T06:18:17,423 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for c151951bebfcfd871997dfedda0e3149 in 560ms, sequenceid=219, compaction requested=true 2024-11-14T06:18:17,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for c151951bebfcfd871997dfedda0e3149: 2024-11-14T06:18:17,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:17,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-14T06:18:17,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-14T06:18:17,425 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-14T06:18:17,426 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9430 sec 2024-11-14T06:18:17,427 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.9480 sec 2024-11-14T06:18:17,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on c151951bebfcfd871997dfedda0e3149 2024-11-14T06:18:17,551 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c151951bebfcfd871997dfedda0e3149 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-14T06:18:17,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=A 2024-11-14T06:18:17,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:17,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=B 2024-11-14T06:18:17,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:17,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=C 2024-11-14T06:18:17,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:17,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/3b89aa76c12a4240ab018e36f3aabdcb is 50, key is test_row_0/A:col10/1731565097191/Put/seqid=0 2024-11-14T06:18:17,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742317_1493 (size=14541) 2024-11-14T06:18:17,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:17,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47634 deadline: 1731565157588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:17,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:17,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:17,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:17,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:17,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47628 deadline: 1731565157589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:17,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:17,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:17,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:17,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-14T06:18:17,627 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:18:17,627 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T06:18:17,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-14T06:18:17,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-14T06:18:17,629 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T06:18:17,630 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T06:18:17,630 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T06:18:17,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:17,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47646 deadline: 1731565157657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:17,665 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to ef93ef3a83c5,42283,1731564965945 for 'test_row_1' in c151951bebfcfd871997dfedda0e3149 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8267 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T06:18:17,666 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:17,666 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:17,666 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:17,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:17,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47634 deadline: 1731565157697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:17,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:17,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:17,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:17,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:17,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47628 deadline: 1731565157707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:17,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:17,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:17,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:17,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:17,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47606 deadline: 1731565157717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:17,723 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to ef93ef3a83c5,42283,1731564965945 for 'test_row_0' in c151951bebfcfd871997dfedda0e3149 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T06:18:17,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:17,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:17,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:17,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-14T06:18:17,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:17,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731565157737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:17,746 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to ef93ef3a83c5,42283,1731564965945 for 'test_row_1' in c151951bebfcfd871997dfedda0e3149 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T06:18:17,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:17,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:17,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:17,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T06:18:17,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:17,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. as already flushing 2024-11-14T06:18:17,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:17,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:17,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:17,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:17,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:17,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47634 deadline: 1731565157908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:17,918 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:17,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:17,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:17,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:17,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47628 deadline: 1731565157917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:17,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:17,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:17,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:17,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T06:18:17,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:17,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. as already flushing 2024-11-14T06:18:17,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:17,935 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:17,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:17,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:17,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-14T06:18:17,958 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/3b89aa76c12a4240ab018e36f3aabdcb 2024-11-14T06:18:17,963 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/da20c0982a1c4ef9b891393d52ebfa64 is 50, key is test_row_0/B:col10/1731565097191/Put/seqid=0 2024-11-14T06:18:17,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742318_1494 (size=12151) 2024-11-14T06:18:18,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T06:18:18,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:18,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. as already flushing 2024-11-14T06:18:18,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:18,087 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:18,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:18,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:18,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:18,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47634 deadline: 1731565158227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:18,232 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:18,232 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:18,232 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:18,240 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T06:18:18,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:18,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. as already flushing 2024-11-14T06:18:18,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:18,240 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:18,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:18,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:18,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:18,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47628 deadline: 1731565158237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:18,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:18,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:18,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:18,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-14T06:18:18,366 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/da20c0982a1c4ef9b891393d52ebfa64 2024-11-14T06:18:18,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/fb31af59de214c51a072c302f1ef8ef6 is 50, key is test_row_0/C:col10/1731565097191/Put/seqid=0 2024-11-14T06:18:18,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742319_1495 (size=12151) 2024-11-14T06:18:18,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T06:18:18,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:18,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. as already flushing 2024-11-14T06:18:18,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:18,393 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:18,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:18,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:18,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T06:18:18,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:18,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. as already flushing 2024-11-14T06:18:18,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:18,545 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:18,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:18,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:18,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T06:18:18,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:18,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. as already flushing 2024-11-14T06:18:18,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:18,698 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:18,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:18,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:18,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:18,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47634 deadline: 1731565158737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:18,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:18,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:18,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:18,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:18,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47628 deadline: 1731565158757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:18,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:18,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:18,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:18,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-14T06:18:18,782 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/fb31af59de214c51a072c302f1ef8ef6 2024-11-14T06:18:18,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/3b89aa76c12a4240ab018e36f3aabdcb as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/3b89aa76c12a4240ab018e36f3aabdcb 2024-11-14T06:18:18,787 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/3b89aa76c12a4240ab018e36f3aabdcb, entries=200, sequenceid=238, filesize=14.2 K 2024-11-14T06:18:18,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/da20c0982a1c4ef9b891393d52ebfa64 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/da20c0982a1c4ef9b891393d52ebfa64 2024-11-14T06:18:18,791 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/da20c0982a1c4ef9b891393d52ebfa64, entries=150, sequenceid=238, filesize=11.9 K 2024-11-14T06:18:18,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/fb31af59de214c51a072c302f1ef8ef6 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/fb31af59de214c51a072c302f1ef8ef6 2024-11-14T06:18:18,794 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/fb31af59de214c51a072c302f1ef8ef6, entries=150, sequenceid=238, filesize=11.9 K 2024-11-14T06:18:18,795 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for c151951bebfcfd871997dfedda0e3149 in 1244ms, sequenceid=238, compaction requested=true 2024-11-14T06:18:18,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c151951bebfcfd871997dfedda0e3149: 2024-11-14T06:18:18,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c151951bebfcfd871997dfedda0e3149:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T06:18:18,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:18,795 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T06:18:18,795 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T06:18:18,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c151951bebfcfd871997dfedda0e3149:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T06:18:18,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:18,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c151951bebfcfd871997dfedda0e3149:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T06:18:18,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:18:18,796 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T06:18:18,796 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53726 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T06:18:18,796 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): c151951bebfcfd871997dfedda0e3149/B is initiating minor compaction (all files) 2024-11-14T06:18:18,796 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): c151951bebfcfd871997dfedda0e3149/A is initiating minor compaction (all files) 2024-11-14T06:18:18,796 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c151951bebfcfd871997dfedda0e3149/B in TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:18,796 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c151951bebfcfd871997dfedda0e3149/A in TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:18,796 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/383b634c434f4d10bb2d6a2709ab7fa1, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/cbdf47e6151b4a488be97aa0873c2ec1, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/77bd2c24b59e43cea71383ae2dacc68f, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/da20c0982a1c4ef9b891393d52ebfa64] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp, totalSize=47.8 K 2024-11-14T06:18:18,796 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/5b48a8b71e314300a7c7df412c365e43, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/07aff5d543a54275a83eb76f4b2378f6, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/89262aa98ef54e3bbe607b97f7aac656, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/3b89aa76c12a4240ab018e36f3aabdcb] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp, totalSize=52.5 K 2024-11-14T06:18:18,797 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b48a8b71e314300a7c7df412c365e43, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731565094049 2024-11-14T06:18:18,797 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 383b634c434f4d10bb2d6a2709ab7fa1, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731565094049 2024-11-14T06:18:18,797 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07aff5d543a54275a83eb76f4b2378f6, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731565094836 2024-11-14T06:18:18,797 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting cbdf47e6151b4a488be97aa0873c2ec1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731565094836 2024-11-14T06:18:18,797 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89262aa98ef54e3bbe607b97f7aac656, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731565095530 2024-11-14T06:18:18,797 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 77bd2c24b59e43cea71383ae2dacc68f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731565095530 2024-11-14T06:18:18,797 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b89aa76c12a4240ab018e36f3aabdcb, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731565097191 2024-11-14T06:18:18,797 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting da20c0982a1c4ef9b891393d52ebfa64, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731565097191 2024-11-14T06:18:18,804 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c151951bebfcfd871997dfedda0e3149#A#compaction#420 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:18:18,805 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/32b364ab38354cf5b0d3e96418f9cd04 is 50, key is test_row_0/A:col10/1731565097191/Put/seqid=0 2024-11-14T06:18:18,805 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c151951bebfcfd871997dfedda0e3149#B#compaction#421 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:18:18,806 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/cc850ec550774274828eef401871b139 is 50, key is test_row_0/B:col10/1731565097191/Put/seqid=0 2024-11-14T06:18:18,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742320_1496 (size=12629) 2024-11-14T06:18:18,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742321_1497 (size=12629) 2024-11-14T06:18:18,824 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/cc850ec550774274828eef401871b139 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/cc850ec550774274828eef401871b139 2024-11-14T06:18:18,831 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c151951bebfcfd871997dfedda0e3149/B of c151951bebfcfd871997dfedda0e3149 into cc850ec550774274828eef401871b139(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:18:18,831 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c151951bebfcfd871997dfedda0e3149: 2024-11-14T06:18:18,831 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., storeName=c151951bebfcfd871997dfedda0e3149/B, priority=12, startTime=1731565098795; duration=0sec 2024-11-14T06:18:18,831 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:18:18,831 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c151951bebfcfd871997dfedda0e3149:B 2024-11-14T06:18:18,831 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T06:18:18,832 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T06:18:18,832 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/32b364ab38354cf5b0d3e96418f9cd04 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/32b364ab38354cf5b0d3e96418f9cd04 2024-11-14T06:18:18,832 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): c151951bebfcfd871997dfedda0e3149/C is initiating minor compaction (all files) 2024-11-14T06:18:18,832 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c151951bebfcfd871997dfedda0e3149/C in TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:18,832 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/d9a086ed6ef9415d8eb9f205550149a4, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/0af4712533054a83b642ec6e8e3a6cb3, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/be3f5aef906349cc8597b0e0c015880e, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/fb31af59de214c51a072c302f1ef8ef6] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp, totalSize=47.8 K 2024-11-14T06:18:18,833 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting d9a086ed6ef9415d8eb9f205550149a4, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731565094049 2024-11-14T06:18:18,833 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 0af4712533054a83b642ec6e8e3a6cb3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731565094836 2024-11-14T06:18:18,834 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting be3f5aef906349cc8597b0e0c015880e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731565095530 2024-11-14T06:18:18,834 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting fb31af59de214c51a072c302f1ef8ef6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731565097191 2024-11-14T06:18:18,836 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c151951bebfcfd871997dfedda0e3149/A of c151951bebfcfd871997dfedda0e3149 into 32b364ab38354cf5b0d3e96418f9cd04(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:18:18,836 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c151951bebfcfd871997dfedda0e3149: 2024-11-14T06:18:18,836 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., storeName=c151951bebfcfd871997dfedda0e3149/A, priority=12, startTime=1731565098795; duration=0sec 2024-11-14T06:18:18,836 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:18,836 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c151951bebfcfd871997dfedda0e3149:A 2024-11-14T06:18:18,843 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c151951bebfcfd871997dfedda0e3149#C#compaction#422 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:18:18,843 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/6501d0fec7c147d989220a89f0755220 is 50, key is test_row_0/C:col10/1731565097191/Put/seqid=0 2024-11-14T06:18:18,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742322_1498 (size=12629) 2024-11-14T06:18:18,850 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T06:18:18,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:18,851 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing c151951bebfcfd871997dfedda0e3149 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-14T06:18:18,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=A 2024-11-14T06:18:18,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:18,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=B 2024-11-14T06:18:18,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:18,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=C 2024-11-14T06:18:18,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:18,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/7c6fde6805434f678e1711a26ee1f81d is 50, key is test_row_0/A:col10/1731565097587/Put/seqid=0 2024-11-14T06:18:18,861 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/6501d0fec7c147d989220a89f0755220 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/6501d0fec7c147d989220a89f0755220 2024-11-14T06:18:18,865 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c151951bebfcfd871997dfedda0e3149/C of c151951bebfcfd871997dfedda0e3149 into 6501d0fec7c147d989220a89f0755220(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:18:18,865 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c151951bebfcfd871997dfedda0e3149: 2024-11-14T06:18:18,865 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., storeName=c151951bebfcfd871997dfedda0e3149/C, priority=12, startTime=1731565098795; duration=0sec 2024-11-14T06:18:18,865 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:18,865 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c151951bebfcfd871997dfedda0e3149:C 2024-11-14T06:18:18,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742323_1499 (size=12151) 2024-11-14T06:18:18,867 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/7c6fde6805434f678e1711a26ee1f81d 2024-11-14T06:18:18,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/03971a5355044dd3bf09732d258b91a9 is 50, key is test_row_0/B:col10/1731565097587/Put/seqid=0 2024-11-14T06:18:18,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742324_1500 (size=12151) 2024-11-14T06:18:18,888 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/03971a5355044dd3bf09732d258b91a9 2024-11-14T06:18:18,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/404a54494afd4084bfed1ca2dac7a83a is 50, key is test_row_0/C:col10/1731565097587/Put/seqid=0 2024-11-14T06:18:18,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742325_1501 (size=12151) 2024-11-14T06:18:19,310 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/404a54494afd4084bfed1ca2dac7a83a 2024-11-14T06:18:19,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/7c6fde6805434f678e1711a26ee1f81d as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/7c6fde6805434f678e1711a26ee1f81d 2024-11-14T06:18:19,325 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/7c6fde6805434f678e1711a26ee1f81d, entries=150, sequenceid=253, filesize=11.9 K 2024-11-14T06:18:19,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/03971a5355044dd3bf09732d258b91a9 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/03971a5355044dd3bf09732d258b91a9 2024-11-14T06:18:19,334 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/03971a5355044dd3bf09732d258b91a9, entries=150, sequenceid=253, filesize=11.9 K 2024-11-14T06:18:19,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/404a54494afd4084bfed1ca2dac7a83a as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/404a54494afd4084bfed1ca2dac7a83a 2024-11-14T06:18:19,347 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/404a54494afd4084bfed1ca2dac7a83a, entries=150, sequenceid=253, filesize=11.9 K 2024-11-14T06:18:19,348 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=0 B/0 for c151951bebfcfd871997dfedda0e3149 in 498ms, sequenceid=253, compaction requested=false 2024-11-14T06:18:19,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for c151951bebfcfd871997dfedda0e3149: 2024-11-14T06:18:19,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:19,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-14T06:18:19,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-14T06:18:19,351 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-14T06:18:19,351 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7190 sec 2024-11-14T06:18:19,353 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.7240 sec 2024-11-14T06:18:19,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on c151951bebfcfd871997dfedda0e3149 2024-11-14T06:18:19,755 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c151951bebfcfd871997dfedda0e3149 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T06:18:19,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=A 2024-11-14T06:18:19,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:19,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=B 2024-11-14T06:18:19,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:19,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=C 2024-11-14T06:18:19,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:19,759 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/ce2c4cf6cf9e4663916e86ceb5308c04 is 50, key is test_row_0/A:col10/1731565099747/Put/seqid=0 2024-11-14T06:18:19,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742326_1502 (size=14741) 2024-11-14T06:18:19,768 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/ce2c4cf6cf9e4663916e86ceb5308c04 2024-11-14T06:18:19,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-14T06:18:19,777 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:18:19,778 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T06:18:19,778 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/d8d6c2691f274da5818ac466dc4de369 is 50, key is test_row_0/B:col10/1731565099747/Put/seqid=0 2024-11-14T06:18:19,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-14T06:18:19,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-14T06:18:19,780 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T06:18:19,781 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T06:18:19,781 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T06:18:19,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742327_1503 (size=12301) 2024-11-14T06:18:19,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731565099791, max timestamp=1731565099812Num uniques 859; 2024-11-14T06:18:19,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731565099791, max timestamp=1731565099812Num uniques 859; 2024-11-14T06:18:19,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T06:18:19,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731565099791, max timestamp=1731565099812Num uniques 859; 2024-11-14T06:18:19,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T06:18:19,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T06:18:19,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:19,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47628 deadline: 1731565159860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:19,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:19,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:19,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:19,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:19,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47606 deadline: 1731565159862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:19,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:19,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:19,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:19,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:19,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47634 deadline: 1731565159876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:19,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:19,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:19,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:19,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:19,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731565159877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:19,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:19,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:19,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:19,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-14T06:18:19,932 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-14T06:18:19,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:19,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. as already flushing 2024-11-14T06:18:19,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:19,933 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:19,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:19,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:19,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:19,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47628 deadline: 1731565159987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:19,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:19,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:19,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:19,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47606 deadline: 1731565159987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:19,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:19,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:19,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:19,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:20,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:20,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47634 deadline: 1731565159997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:20,006 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:20,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:20,006 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:20,006 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:20,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731565159997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:20,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:20,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:20,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:20,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-14T06:18:20,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:20,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. as already flushing 2024-11-14T06:18:20,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:20,085 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:20,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:20,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:20,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-14T06:18:20,198 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/d8d6c2691f274da5818ac466dc4de369 2024-11-14T06:18:20,203 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/b4ecd4625d4e4c7e85f535d7e221aa86 is 50, key is test_row_0/C:col10/1731565099747/Put/seqid=0 2024-11-14T06:18:20,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:20,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47628 deadline: 1731565160197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:20,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:20,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:20,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47606 deadline: 1731565160197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:20,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:20,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:20,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:20,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:20,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:20,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742328_1504 (size=12301) 2024-11-14T06:18:20,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:20,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47634 deadline: 1731565160217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:20,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:20,227 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:20,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731565160218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:20,227 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:20,227 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:20,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:20,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:20,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:20,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-14T06:18:20,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:20,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. as already flushing 2024-11-14T06:18:20,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:20,238 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:20,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:20,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:20,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-14T06:18:20,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:20,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. as already flushing 2024-11-14T06:18:20,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:20,390 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:20,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:20,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:20,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-14T06:18:20,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:20,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47628 deadline: 1731565160517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:20,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:20,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47606 deadline: 1731565160517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:20,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:20,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:20,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:20,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:20,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:20,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:20,542 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-14T06:18:20,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:20,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. as already flushing 2024-11-14T06:18:20,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:20,543 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:20,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:20,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:20,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:20,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47634 deadline: 1731565160537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:20,546 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:20,546 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:20,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:20,546 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:20,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731565160537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:20,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:20,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:20,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:20,608 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/b4ecd4625d4e4c7e85f535d7e221aa86 2024-11-14T06:18:20,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/ce2c4cf6cf9e4663916e86ceb5308c04 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/ce2c4cf6cf9e4663916e86ceb5308c04 2024-11-14T06:18:20,615 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/ce2c4cf6cf9e4663916e86ceb5308c04, entries=200, sequenceid=264, filesize=14.4 K 2024-11-14T06:18:20,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/d8d6c2691f274da5818ac466dc4de369 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/d8d6c2691f274da5818ac466dc4de369 2024-11-14T06:18:20,618 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/d8d6c2691f274da5818ac466dc4de369, entries=150, sequenceid=264, filesize=12.0 K 2024-11-14T06:18:20,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/b4ecd4625d4e4c7e85f535d7e221aa86 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/b4ecd4625d4e4c7e85f535d7e221aa86 2024-11-14T06:18:20,621 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/b4ecd4625d4e4c7e85f535d7e221aa86, entries=150, sequenceid=264, filesize=12.0 K 2024-11-14T06:18:20,621 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for c151951bebfcfd871997dfedda0e3149 in 866ms, sequenceid=264, compaction requested=true 2024-11-14T06:18:20,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c151951bebfcfd871997dfedda0e3149: 2024-11-14T06:18:20,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c151951bebfcfd871997dfedda0e3149:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T06:18:20,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:20,622 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:18:20,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c151951bebfcfd871997dfedda0e3149:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T06:18:20,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:20,622 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:18:20,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c151951bebfcfd871997dfedda0e3149:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T06:18:20,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:18:20,622 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39521 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:18:20,622 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37081 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:18:20,622 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): c151951bebfcfd871997dfedda0e3149/A is initiating minor compaction (all files) 2024-11-14T06:18:20,622 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): c151951bebfcfd871997dfedda0e3149/B is initiating minor compaction (all files) 2024-11-14T06:18:20,622 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c151951bebfcfd871997dfedda0e3149/A in TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:20,622 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c151951bebfcfd871997dfedda0e3149/B in TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:20,622 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/32b364ab38354cf5b0d3e96418f9cd04, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/7c6fde6805434f678e1711a26ee1f81d, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/ce2c4cf6cf9e4663916e86ceb5308c04] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp, totalSize=38.6 K 2024-11-14T06:18:20,622 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/cc850ec550774274828eef401871b139, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/03971a5355044dd3bf09732d258b91a9, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/d8d6c2691f274da5818ac466dc4de369] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp, totalSize=36.2 K 2024-11-14T06:18:20,623 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32b364ab38354cf5b0d3e96418f9cd04, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731565097191 2024-11-14T06:18:20,623 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting cc850ec550774274828eef401871b139, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731565097191 2024-11-14T06:18:20,623 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 03971a5355044dd3bf09732d258b91a9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731565097579 2024-11-14T06:18:20,623 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c6fde6805434f678e1711a26ee1f81d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731565097579 2024-11-14T06:18:20,623 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting d8d6c2691f274da5818ac466dc4de369, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731565099747 2024-11-14T06:18:20,623 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce2c4cf6cf9e4663916e86ceb5308c04, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731565099743 2024-11-14T06:18:20,630 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c151951bebfcfd871997dfedda0e3149#A#compaction#429 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:18:20,630 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/2217513902b34c218d919ad0596f7527 is 50, key is test_row_0/A:col10/1731565099747/Put/seqid=0 2024-11-14T06:18:20,632 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c151951bebfcfd871997dfedda0e3149#B#compaction#430 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:18:20,633 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/94e40894ba4a430691f23183c028d1c9 is 50, key is test_row_0/B:col10/1731565099747/Put/seqid=0 2024-11-14T06:18:20,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742329_1505 (size=12881) 2024-11-14T06:18:20,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742330_1506 (size=12881) 2024-11-14T06:18:20,643 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/94e40894ba4a430691f23183c028d1c9 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/94e40894ba4a430691f23183c028d1c9 2024-11-14T06:18:20,658 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c151951bebfcfd871997dfedda0e3149/B of c151951bebfcfd871997dfedda0e3149 into 94e40894ba4a430691f23183c028d1c9(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:18:20,658 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c151951bebfcfd871997dfedda0e3149: 2024-11-14T06:18:20,658 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., storeName=c151951bebfcfd871997dfedda0e3149/B, priority=13, startTime=1731565100622; duration=0sec 2024-11-14T06:18:20,658 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:18:20,658 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c151951bebfcfd871997dfedda0e3149:B 2024-11-14T06:18:20,658 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:18:20,662 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37081 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:18:20,663 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): c151951bebfcfd871997dfedda0e3149/C is initiating minor compaction (all files) 2024-11-14T06:18:20,663 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c151951bebfcfd871997dfedda0e3149/C in TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:20,663 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/6501d0fec7c147d989220a89f0755220, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/404a54494afd4084bfed1ca2dac7a83a, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/b4ecd4625d4e4c7e85f535d7e221aa86] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp, totalSize=36.2 K 2024-11-14T06:18:20,663 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 6501d0fec7c147d989220a89f0755220, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731565097191 2024-11-14T06:18:20,663 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 404a54494afd4084bfed1ca2dac7a83a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731565097579 2024-11-14T06:18:20,664 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting b4ecd4625d4e4c7e85f535d7e221aa86, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731565099747 2024-11-14T06:18:20,670 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c151951bebfcfd871997dfedda0e3149#C#compaction#431 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:18:20,671 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/5aa2005443de4ce9aafd5620dc757f7e is 50, key is test_row_0/C:col10/1731565099747/Put/seqid=0 2024-11-14T06:18:20,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742331_1507 (size=12881) 2024-11-14T06:18:20,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-14T06:18:20,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:20,696 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing c151951bebfcfd871997dfedda0e3149 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-14T06:18:20,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=A 2024-11-14T06:18:20,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:18:20,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=B 2024-11-14T06:18:20,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:18:20,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=C 2024-11-14T06:18:20,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:18:20,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/f09e3a263161483f9f15a555d3e1151c is 50, key is test_row_0/A:col10/1731565099876/Put/seqid=0 2024-11-14T06:18:20,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742332_1508 (size=12301) 2024-11-14T06:18:20,707 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/f09e3a263161483f9f15a555d3e1151c 2024-11-14T06:18:20,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/755090207794465686b0ef183a8bb05f is 50, key is test_row_0/B:col10/1731565099876/Put/seqid=0 2024-11-14T06:18:20,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742333_1509 (size=12301) 2024-11-14T06:18:20,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-14T06:18:21,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on c151951bebfcfd871997dfedda0e3149 2024-11-14T06:18:21,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. as already flushing 2024-11-14T06:18:21,043 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/2217513902b34c218d919ad0596f7527 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/2217513902b34c218d919ad0596f7527 2024-11-14T06:18:21,047 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c151951bebfcfd871997dfedda0e3149/A of c151951bebfcfd871997dfedda0e3149 into 2217513902b34c218d919ad0596f7527(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:18:21,047 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c151951bebfcfd871997dfedda0e3149: 2024-11-14T06:18:21,047 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., storeName=c151951bebfcfd871997dfedda0e3149/A, priority=13, startTime=1731565100622; duration=0sec 2024-11-14T06:18:21,047 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:21,047 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c151951bebfcfd871997dfedda0e3149:A 2024-11-14T06:18:21,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:21,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47606 deadline: 1731565161052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:21,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:21,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:21,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:21,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:21,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731565161057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:21,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:21,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:21,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:21,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:21,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47634 deadline: 1731565161061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:21,068 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:21,068 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:21,068 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:21,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:21,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47628 deadline: 1731565161062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:21,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:21,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:21,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:21,081 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/5aa2005443de4ce9aafd5620dc757f7e as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/5aa2005443de4ce9aafd5620dc757f7e 2024-11-14T06:18:21,084 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c151951bebfcfd871997dfedda0e3149/C of c151951bebfcfd871997dfedda0e3149 into 5aa2005443de4ce9aafd5620dc757f7e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:18:21,084 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c151951bebfcfd871997dfedda0e3149: 2024-11-14T06:18:21,084 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., storeName=c151951bebfcfd871997dfedda0e3149/C, priority=13, startTime=1731565100622; duration=0sec 2024-11-14T06:18:21,084 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:21,084 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c151951bebfcfd871997dfedda0e3149:C 2024-11-14T06:18:21,119 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/755090207794465686b0ef183a8bb05f 2024-11-14T06:18:21,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/6c236b2a083f41d2998bd7d5090ddc88 is 50, key is test_row_0/C:col10/1731565099876/Put/seqid=0 2024-11-14T06:18:21,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742334_1510 (size=12301) 2024-11-14T06:18:21,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:21,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47606 deadline: 1731565161167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:21,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:21,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:21,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:21,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:21,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47628 deadline: 1731565161185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:21,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:21,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:21,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47634 deadline: 1731565161185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:21,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:21,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:21,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:21,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:21,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:21,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:21,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47606 deadline: 1731565161387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:21,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:21,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:21,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:21,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:21,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47628 deadline: 1731565161408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:21,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:21,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:21,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47634 deadline: 1731565161408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:21,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:21,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:21,421 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:21,421 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:21,421 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:21,528 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/6c236b2a083f41d2998bd7d5090ddc88 2024-11-14T06:18:21,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/f09e3a263161483f9f15a555d3e1151c as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/f09e3a263161483f9f15a555d3e1151c 2024-11-14T06:18:21,542 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/f09e3a263161483f9f15a555d3e1151c, entries=150, sequenceid=300, filesize=12.0 K 2024-11-14T06:18:21,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/755090207794465686b0ef183a8bb05f as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/755090207794465686b0ef183a8bb05f 2024-11-14T06:18:21,547 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/755090207794465686b0ef183a8bb05f, entries=150, sequenceid=300, filesize=12.0 K 2024-11-14T06:18:21,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/6c236b2a083f41d2998bd7d5090ddc88 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/6c236b2a083f41d2998bd7d5090ddc88 2024-11-14T06:18:21,551 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/6c236b2a083f41d2998bd7d5090ddc88, entries=150, sequenceid=300, filesize=12.0 K 2024-11-14T06:18:21,552 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for c151951bebfcfd871997dfedda0e3149 in 856ms, sequenceid=300, compaction requested=false 2024-11-14T06:18:21,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for c151951bebfcfd871997dfedda0e3149: 2024-11-14T06:18:21,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:21,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-14T06:18:21,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-14T06:18:21,558 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-14T06:18:21,558 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7750 sec 2024-11-14T06:18:21,560 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.7810 sec 2024-11-14T06:18:21,713 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c151951bebfcfd871997dfedda0e3149 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-14T06:18:21,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=A 2024-11-14T06:18:21,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:21,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=B 2024-11-14T06:18:21,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:21,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=C 2024-11-14T06:18:21,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:21,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on c151951bebfcfd871997dfedda0e3149 2024-11-14T06:18:21,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/0dd4c9f1aad14d63a9f70ddb6f8bfd15 is 50, key is test_row_0/A:col10/1731565101043/Put/seqid=0 2024-11-14T06:18:21,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742335_1511 (size=14741) 2024-11-14T06:18:21,733 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/0dd4c9f1aad14d63a9f70ddb6f8bfd15 2024-11-14T06:18:21,747 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/5649e1ca1bd2495cbb427045dc20f48b is 50, key is test_row_0/B:col10/1731565101043/Put/seqid=0 2024-11-14T06:18:21,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742336_1512 (size=12301) 2024-11-14T06:18:21,812 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731565101772, max timestamp=1731565101809Num uniques 859; 2024-11-14T06:18:21,812 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T06:18:21,812 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731565101772, max timestamp=1731565101809Num uniques 859; 2024-11-14T06:18:21,812 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731565101772, max timestamp=1731565101809Num uniques 859; 2024-11-14T06:18:21,812 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T06:18:21,812 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T06:18:21,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:21,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47634 deadline: 1731565161863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:21,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:21,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:21,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:21,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:21,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47628 deadline: 1731565161865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:21,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:21,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:21,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:21,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:21,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47606 deadline: 1731565161871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:21,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:21,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:21,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:21,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-14T06:18:21,928 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:18:21,928 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T06:18:21,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-14T06:18:21,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-14T06:18:21,930 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T06:18:21,931 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T06:18:21,931 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T06:18:21,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:21,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47634 deadline: 1731565161978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:21,996 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:21,996 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:21,996 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:22,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:22,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47628 deadline: 1731565161987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:22,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:22,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47606 deadline: 1731565161991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:22,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:22,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:22,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:22,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:22,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:22,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:22,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-14T06:18:22,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:22,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731565162077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:22,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-14T06:18:22,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:22,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. as already flushing 2024-11-14T06:18:22,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:22,084 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:22,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:22,084 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to ef93ef3a83c5,42283,1731564965945 for 'test_row_1' in c151951bebfcfd871997dfedda0e3149 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T06:18:22,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:22,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:22,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:22,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:22,168 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/5649e1ca1bd2495cbb427045dc20f48b 2024-11-14T06:18:22,174 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/5f443cd7ac074339849dd7f1d0aa476b is 50, key is test_row_0/C:col10/1731565101043/Put/seqid=0 2024-11-14T06:18:22,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742337_1513 (size=12301) 2024-11-14T06:18:22,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:22,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47634 deadline: 1731565162207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:22,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:22,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:22,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:22,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:22,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47606 deadline: 1731565162207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:22,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:22,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:22,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:22,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:22,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47628 deadline: 1731565162208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:22,212 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:22,212 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:22,212 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:22,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-14T06:18:22,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:22,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. as already flushing 2024-11-14T06:18:22,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:22,236 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:22,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:22,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:22,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-14T06:18:22,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-14T06:18:22,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:22,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. as already flushing 2024-11-14T06:18:22,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:22,389 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:22,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:22,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:22,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:22,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47634 deadline: 1731565162517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:22,525 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:22,525 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:22,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:22,525 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:22,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47606 deadline: 1731565162518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:22,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:22,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:22,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:22,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:22,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47628 deadline: 1731565162518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:22,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:22,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:22,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:22,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-14T06:18:22,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:22,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. as already flushing 2024-11-14T06:18:22,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:22,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:22,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:22,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:22,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-14T06:18:22,586 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/5f443cd7ac074339849dd7f1d0aa476b 2024-11-14T06:18:22,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/0dd4c9f1aad14d63a9f70ddb6f8bfd15 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/0dd4c9f1aad14d63a9f70ddb6f8bfd15 2024-11-14T06:18:22,592 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/0dd4c9f1aad14d63a9f70ddb6f8bfd15, entries=200, sequenceid=313, filesize=14.4 K 2024-11-14T06:18:22,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/5649e1ca1bd2495cbb427045dc20f48b as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/5649e1ca1bd2495cbb427045dc20f48b 2024-11-14T06:18:22,598 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/5649e1ca1bd2495cbb427045dc20f48b, entries=150, sequenceid=313, filesize=12.0 K 2024-11-14T06:18:22,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/5f443cd7ac074339849dd7f1d0aa476b as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/5f443cd7ac074339849dd7f1d0aa476b 2024-11-14T06:18:22,601 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/5f443cd7ac074339849dd7f1d0aa476b, entries=150, sequenceid=313, filesize=12.0 K 2024-11-14T06:18:22,602 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for c151951bebfcfd871997dfedda0e3149 in 889ms, sequenceid=313, compaction requested=true 2024-11-14T06:18:22,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c151951bebfcfd871997dfedda0e3149: 2024-11-14T06:18:22,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c151951bebfcfd871997dfedda0e3149:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T06:18:22,602 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:18:22,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:22,602 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:18:22,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c151951bebfcfd871997dfedda0e3149:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T06:18:22,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:22,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c151951bebfcfd871997dfedda0e3149:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T06:18:22,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:18:22,603 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39923 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:18:22,603 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): c151951bebfcfd871997dfedda0e3149/A is initiating minor compaction (all files) 2024-11-14T06:18:22,603 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c151951bebfcfd871997dfedda0e3149/A in TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:22,603 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/2217513902b34c218d919ad0596f7527, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/f09e3a263161483f9f15a555d3e1151c, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/0dd4c9f1aad14d63a9f70ddb6f8bfd15] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp, totalSize=39.0 K 2024-11-14T06:18:22,604 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:18:22,604 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): c151951bebfcfd871997dfedda0e3149/B is initiating minor compaction (all files) 2024-11-14T06:18:22,604 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2217513902b34c218d919ad0596f7527, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731565099747 2024-11-14T06:18:22,604 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c151951bebfcfd871997dfedda0e3149/B in TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:22,604 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/94e40894ba4a430691f23183c028d1c9, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/755090207794465686b0ef183a8bb05f, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/5649e1ca1bd2495cbb427045dc20f48b] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp, totalSize=36.6 K 2024-11-14T06:18:22,604 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting f09e3a263161483f9f15a555d3e1151c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731565099858 2024-11-14T06:18:22,604 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 94e40894ba4a430691f23183c028d1c9, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731565099747 2024-11-14T06:18:22,604 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0dd4c9f1aad14d63a9f70ddb6f8bfd15, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731565101043 2024-11-14T06:18:22,604 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 755090207794465686b0ef183a8bb05f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731565099858 2024-11-14T06:18:22,605 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 5649e1ca1bd2495cbb427045dc20f48b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731565101043 2024-11-14T06:18:22,614 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c151951bebfcfd871997dfedda0e3149#B#compaction#439 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:18:22,614 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c151951bebfcfd871997dfedda0e3149#A#compaction#438 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:18:22,614 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/9a39cdd4016c43198ad0798516026b36 is 50, key is test_row_0/B:col10/1731565101043/Put/seqid=0 2024-11-14T06:18:22,614 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/93f445866d1144ea85efcbcdbcb8a22d is 50, key is test_row_0/A:col10/1731565101043/Put/seqid=0 2024-11-14T06:18:22,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742338_1514 (size=12983) 2024-11-14T06:18:22,638 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/9a39cdd4016c43198ad0798516026b36 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/9a39cdd4016c43198ad0798516026b36 2024-11-14T06:18:22,646 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c151951bebfcfd871997dfedda0e3149/B of c151951bebfcfd871997dfedda0e3149 into 9a39cdd4016c43198ad0798516026b36(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:18:22,646 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c151951bebfcfd871997dfedda0e3149: 2024-11-14T06:18:22,646 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., storeName=c151951bebfcfd871997dfedda0e3149/B, priority=13, startTime=1731565102602; duration=0sec 2024-11-14T06:18:22,646 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:18:22,646 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c151951bebfcfd871997dfedda0e3149:B 2024-11-14T06:18:22,646 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:18:22,647 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:18:22,647 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): c151951bebfcfd871997dfedda0e3149/C is initiating minor compaction (all files) 2024-11-14T06:18:22,647 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c151951bebfcfd871997dfedda0e3149/C in TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:22,647 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/5aa2005443de4ce9aafd5620dc757f7e, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/6c236b2a083f41d2998bd7d5090ddc88, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/5f443cd7ac074339849dd7f1d0aa476b] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp, totalSize=36.6 K 2024-11-14T06:18:22,647 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 5aa2005443de4ce9aafd5620dc757f7e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731565099747 2024-11-14T06:18:22,648 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c236b2a083f41d2998bd7d5090ddc88, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731565099858 2024-11-14T06:18:22,648 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f443cd7ac074339849dd7f1d0aa476b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731565101043 2024-11-14T06:18:22,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742339_1515 (size=12983) 2024-11-14T06:18:22,656 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c151951bebfcfd871997dfedda0e3149#C#compaction#440 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:18:22,657 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/0e8b15b1207646fda27aefdde022cfbd is 50, key is test_row_0/C:col10/1731565101043/Put/seqid=0 2024-11-14T06:18:22,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742340_1516 (size=12983) 2024-11-14T06:18:22,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-14T06:18:22,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:22,694 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing c151951bebfcfd871997dfedda0e3149 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-14T06:18:22,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=A 2024-11-14T06:18:22,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:18:22,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=B 2024-11-14T06:18:22,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:18:22,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=C 2024-11-14T06:18:22,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:18:22,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/38f700fc50784855b30a356c471dbfa8 is 50, key is test_row_0/A:col10/1731565101869/Put/seqid=0 2024-11-14T06:18:22,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742341_1517 (size=12301) 2024-11-14T06:18:22,711 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/38f700fc50784855b30a356c471dbfa8 2024-11-14T06:18:22,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/d9239c85eac24f57a8fc6c391532b8b2 is 50, key is test_row_0/B:col10/1731565101869/Put/seqid=0 2024-11-14T06:18:22,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742342_1518 (size=12301) 2024-11-14T06:18:22,729 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/d9239c85eac24f57a8fc6c391532b8b2 2024-11-14T06:18:22,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/02fac31d63f64f77a065a711ff106ad7 is 50, key is test_row_0/C:col10/1731565101869/Put/seqid=0 2024-11-14T06:18:22,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742343_1519 (size=12301) 2024-11-14T06:18:22,754 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/02fac31d63f64f77a065a711ff106ad7 2024-11-14T06:18:22,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/38f700fc50784855b30a356c471dbfa8 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/38f700fc50784855b30a356c471dbfa8 2024-11-14T06:18:22,765 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/38f700fc50784855b30a356c471dbfa8, entries=150, sequenceid=349, filesize=12.0 K 2024-11-14T06:18:22,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/d9239c85eac24f57a8fc6c391532b8b2 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/d9239c85eac24f57a8fc6c391532b8b2 2024-11-14T06:18:22,778 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/d9239c85eac24f57a8fc6c391532b8b2, entries=150, sequenceid=349, filesize=12.0 K 2024-11-14T06:18:22,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/02fac31d63f64f77a065a711ff106ad7 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/02fac31d63f64f77a065a711ff106ad7 2024-11-14T06:18:22,791 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/02fac31d63f64f77a065a711ff106ad7, entries=150, sequenceid=349, filesize=12.0 K 2024-11-14T06:18:22,792 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=0 B/0 for c151951bebfcfd871997dfedda0e3149 in 98ms, sequenceid=349, compaction requested=false 2024-11-14T06:18:22,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for c151951bebfcfd871997dfedda0e3149: 2024-11-14T06:18:22,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:22,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-14T06:18:22,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-14T06:18:22,798 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-14T06:18:22,798 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 865 msec 2024-11-14T06:18:22,799 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 870 msec 2024-11-14T06:18:23,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on c151951bebfcfd871997dfedda0e3149 2024-11-14T06:18:23,038 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c151951bebfcfd871997dfedda0e3149 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T06:18:23,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=A 2024-11-14T06:18:23,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:23,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=B 2024-11-14T06:18:23,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:23,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=C 2024-11-14T06:18:23,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:23,042 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/b3e063d5a2124f7199d60c048bc03bbf is 50, key is test_row_0/A:col10/1731565103037/Put/seqid=0 2024-11-14T06:18:23,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742344_1520 (size=17181) 2024-11-14T06:18:23,060 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/b3e063d5a2124f7199d60c048bc03bbf 2024-11-14T06:18:23,062 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/93f445866d1144ea85efcbcdbcb8a22d as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/93f445866d1144ea85efcbcdbcb8a22d 2024-11-14T06:18:23,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-14T06:18:23,067 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c151951bebfcfd871997dfedda0e3149/A of c151951bebfcfd871997dfedda0e3149 into 93f445866d1144ea85efcbcdbcb8a22d(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:18:23,067 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c151951bebfcfd871997dfedda0e3149: 2024-11-14T06:18:23,067 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., storeName=c151951bebfcfd871997dfedda0e3149/A, priority=13, startTime=1731565102602; duration=0sec 2024-11-14T06:18:23,067 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:18:23,067 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:23,067 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c151951bebfcfd871997dfedda0e3149:A 2024-11-14T06:18:23,067 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T06:18:23,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-14T06:18:23,069 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/69d8fb2084134abbbfb7422291c874f6 is 50, key is test_row_0/B:col10/1731565103037/Put/seqid=0 2024-11-14T06:18:23,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-14T06:18:23,071 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T06:18:23,072 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T06:18:23,072 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T06:18:23,081 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/0e8b15b1207646fda27aefdde022cfbd as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/0e8b15b1207646fda27aefdde022cfbd 2024-11-14T06:18:23,086 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c151951bebfcfd871997dfedda0e3149/C of c151951bebfcfd871997dfedda0e3149 into 0e8b15b1207646fda27aefdde022cfbd(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:18:23,086 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c151951bebfcfd871997dfedda0e3149: 2024-11-14T06:18:23,086 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., storeName=c151951bebfcfd871997dfedda0e3149/C, priority=13, startTime=1731565102603; duration=0sec 2024-11-14T06:18:23,086 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:23,086 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c151951bebfcfd871997dfedda0e3149:C 2024-11-14T06:18:23,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742345_1521 (size=12301) 2024-11-14T06:18:23,110 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731565103087, max timestamp=1731565103108Num uniques 859; 2024-11-14T06:18:23,110 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731565103087, max timestamp=1731565103108Num uniques 859; 2024-11-14T06:18:23,110 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T06:18:23,110 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T06:18:23,111 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731565103087, max timestamp=1731565103108Num uniques 859; 2024-11-14T06:18:23,111 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T06:18:23,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-14T06:18:23,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:23,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47606 deadline: 1731565163178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:23,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:23,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:23,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:23,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:23,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47634 deadline: 1731565163178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:23,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:23,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:23,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:23,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:23,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47628 deadline: 1731565163189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:23,201 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:23,201 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:23,201 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:23,224 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-14T06:18:23,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:23,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. as already flushing 2024-11-14T06:18:23,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:23,225 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:23,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:23,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:23,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:23,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47606 deadline: 1731565163297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:23,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:23,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:23,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:23,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:23,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47634 deadline: 1731565163307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:23,316 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:23,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:23,316 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:23,316 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:23,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47628 deadline: 1731565163307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:23,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:23,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:23,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:23,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-14T06:18:23,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:23,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. as already flushing 2024-11-14T06:18:23,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:23,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:23,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:23,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:23,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-14T06:18:23,490 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/69d8fb2084134abbbfb7422291c874f6 2024-11-14T06:18:23,512 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/45c2c0c746714f1e8791d00ce906312e is 50, key is test_row_0/C:col10/1731565103037/Put/seqid=0 2024-11-14T06:18:23,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:23,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47606 deadline: 1731565163517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:23,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:23,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:23,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:23,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742346_1522 (size=12301) 2024-11-14T06:18:23,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-14T06:18:23,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:23,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. as already flushing 2024-11-14T06:18:23,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:23,532 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:23,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:23,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:23,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:23,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47634 deadline: 1731565163527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:23,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:23,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:23,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:23,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47628 deadline: 1731565163527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:23,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:23,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:23,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:23,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:23,686 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-14T06:18:23,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:23,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. as already flushing 2024-11-14T06:18:23,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:23,686 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:23,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:23,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:23,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-14T06:18:23,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:23,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47606 deadline: 1731565163828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:23,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-14T06:18:23,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:23,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:23,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:23,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:23,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. as already flushing 2024-11-14T06:18:23,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:23,839 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:23,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:23,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:23,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:23,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47634 deadline: 1731565163847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:23,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:23,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47628 deadline: 1731565163847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:23,856 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:23,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:23,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:23,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:23,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:23,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:23,931 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/45c2c0c746714f1e8791d00ce906312e 2024-11-14T06:18:23,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/b3e063d5a2124f7199d60c048bc03bbf as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/b3e063d5a2124f7199d60c048bc03bbf 2024-11-14T06:18:23,938 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/b3e063d5a2124f7199d60c048bc03bbf, entries=250, sequenceid=359, filesize=16.8 K 2024-11-14T06:18:23,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/69d8fb2084134abbbfb7422291c874f6 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/69d8fb2084134abbbfb7422291c874f6 2024-11-14T06:18:23,948 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/69d8fb2084134abbbfb7422291c874f6, entries=150, sequenceid=359, filesize=12.0 K 2024-11-14T06:18:23,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/45c2c0c746714f1e8791d00ce906312e as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/45c2c0c746714f1e8791d00ce906312e 2024-11-14T06:18:23,952 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/45c2c0c746714f1e8791d00ce906312e, entries=150, sequenceid=359, filesize=12.0 K 2024-11-14T06:18:23,954 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for c151951bebfcfd871997dfedda0e3149 in 915ms, sequenceid=359, compaction requested=true 2024-11-14T06:18:23,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c151951bebfcfd871997dfedda0e3149: 2024-11-14T06:18:23,954 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:18:23,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c151951bebfcfd871997dfedda0e3149:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T06:18:23,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:23,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c151951bebfcfd871997dfedda0e3149:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T06:18:23,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:23,954 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:18:23,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c151951bebfcfd871997dfedda0e3149:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T06:18:23,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:18:23,963 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:18:23,963 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): c151951bebfcfd871997dfedda0e3149/B is initiating minor compaction (all files) 2024-11-14T06:18:23,964 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c151951bebfcfd871997dfedda0e3149/B in TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:23,964 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/9a39cdd4016c43198ad0798516026b36, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/d9239c85eac24f57a8fc6c391532b8b2, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/69d8fb2084134abbbfb7422291c874f6] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp, totalSize=36.7 K 2024-11-14T06:18:23,964 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42465 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:18:23,964 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): c151951bebfcfd871997dfedda0e3149/A is initiating minor compaction (all files) 2024-11-14T06:18:23,964 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c151951bebfcfd871997dfedda0e3149/A in TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:23,964 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a39cdd4016c43198ad0798516026b36, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731565101043 2024-11-14T06:18:23,964 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/93f445866d1144ea85efcbcdbcb8a22d, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/38f700fc50784855b30a356c471dbfa8, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/b3e063d5a2124f7199d60c048bc03bbf] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp, totalSize=41.5 K 2024-11-14T06:18:23,964 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting d9239c85eac24f57a8fc6c391532b8b2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1731565101849 2024-11-14T06:18:23,972 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93f445866d1144ea85efcbcdbcb8a22d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731565101043 2024-11-14T06:18:23,973 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 69d8fb2084134abbbfb7422291c874f6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1731565103035 2024-11-14T06:18:23,973 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38f700fc50784855b30a356c471dbfa8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1731565101849 2024-11-14T06:18:23,973 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3e063d5a2124f7199d60c048bc03bbf, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1731565103028 2024-11-14T06:18:23,985 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c151951bebfcfd871997dfedda0e3149#B#compaction#447 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:18:23,985 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/b7c392ff813c494cab23d5306835e3a2 is 50, key is test_row_0/B:col10/1731565103037/Put/seqid=0 2024-11-14T06:18:23,988 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c151951bebfcfd871997dfedda0e3149#A#compaction#448 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:18:23,989 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/6002b7bb708e462c8b1237790e60db13 is 50, key is test_row_0/A:col10/1731565103037/Put/seqid=0 2024-11-14T06:18:23,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-14T06:18:23,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:23,992 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing c151951bebfcfd871997dfedda0e3149 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-14T06:18:23,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=A 2024-11-14T06:18:23,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:18:23,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=B 2024-11-14T06:18:23,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:18:23,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=C 2024-11-14T06:18:23,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:18:24,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742347_1523 (size=13085) 2024-11-14T06:18:24,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/08ee0adeb2e74f1db18f5a0d451026f7 is 50, key is test_row_0/A:col10/1731565103135/Put/seqid=0 2024-11-14T06:18:24,016 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/b7c392ff813c494cab23d5306835e3a2 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/b7c392ff813c494cab23d5306835e3a2 2024-11-14T06:18:24,020 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c151951bebfcfd871997dfedda0e3149/B of c151951bebfcfd871997dfedda0e3149 into b7c392ff813c494cab23d5306835e3a2(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:18:24,020 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c151951bebfcfd871997dfedda0e3149: 2024-11-14T06:18:24,020 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., storeName=c151951bebfcfd871997dfedda0e3149/B, priority=13, startTime=1731565103954; duration=0sec 2024-11-14T06:18:24,020 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:18:24,020 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c151951bebfcfd871997dfedda0e3149:B 2024-11-14T06:18:24,021 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:18:24,034 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:18:24,034 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): c151951bebfcfd871997dfedda0e3149/C is initiating minor compaction (all files) 2024-11-14T06:18:24,034 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c151951bebfcfd871997dfedda0e3149/C in TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:24,034 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/0e8b15b1207646fda27aefdde022cfbd, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/02fac31d63f64f77a065a711ff106ad7, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/45c2c0c746714f1e8791d00ce906312e] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp, totalSize=36.7 K 2024-11-14T06:18:24,034 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e8b15b1207646fda27aefdde022cfbd, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731565101043 2024-11-14T06:18:24,035 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 02fac31d63f64f77a065a711ff106ad7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1731565101849 2024-11-14T06:18:24,035 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 45c2c0c746714f1e8791d00ce906312e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1731565103035 2024-11-14T06:18:24,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742348_1524 (size=13085) 2024-11-14T06:18:24,048 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c151951bebfcfd871997dfedda0e3149#C#compaction#450 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:18:24,048 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/66dd04b4cdae49748fc5133ffecbea8a is 50, key is test_row_0/C:col10/1731565103037/Put/seqid=0 2024-11-14T06:18:24,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742349_1525 (size=12301) 2024-11-14T06:18:24,063 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/6002b7bb708e462c8b1237790e60db13 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/6002b7bb708e462c8b1237790e60db13 2024-11-14T06:18:24,067 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c151951bebfcfd871997dfedda0e3149/A of c151951bebfcfd871997dfedda0e3149 into 6002b7bb708e462c8b1237790e60db13(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:18:24,067 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c151951bebfcfd871997dfedda0e3149: 2024-11-14T06:18:24,067 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., storeName=c151951bebfcfd871997dfedda0e3149/A, priority=13, startTime=1731565103954; duration=0sec 2024-11-14T06:18:24,067 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:24,067 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c151951bebfcfd871997dfedda0e3149:A 2024-11-14T06:18:24,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742350_1526 (size=13085) 2024-11-14T06:18:24,090 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/66dd04b4cdae49748fc5133ffecbea8a as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/66dd04b4cdae49748fc5133ffecbea8a 2024-11-14T06:18:24,097 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c151951bebfcfd871997dfedda0e3149/C of c151951bebfcfd871997dfedda0e3149 into 66dd04b4cdae49748fc5133ffecbea8a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:18:24,097 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c151951bebfcfd871997dfedda0e3149: 2024-11-14T06:18:24,097 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., storeName=c151951bebfcfd871997dfedda0e3149/C, priority=13, startTime=1731565103954; duration=0sec 2024-11-14T06:18:24,097 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:24,097 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c151951bebfcfd871997dfedda0e3149:C 2024-11-14T06:18:24,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on c151951bebfcfd871997dfedda0e3149 2024-11-14T06:18:24,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. as already flushing 2024-11-14T06:18:24,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:24,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731565164174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:24,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:24,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:24,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:24,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-14T06:18:24,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:24,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731565164297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:24,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:24,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:24,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:24,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:24,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47606 deadline: 1731565164347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:24,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:24,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:24,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:24,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:24,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47634 deadline: 1731565164367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:24,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:24,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:24,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:24,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:24,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47628 deadline: 1731565164367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:24,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:24,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:24,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:24,458 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/08ee0adeb2e74f1db18f5a0d451026f7 2024-11-14T06:18:24,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/7c0b7a6e46464c7a92e73e0f615a057a is 50, key is test_row_0/B:col10/1731565103135/Put/seqid=0 2024-11-14T06:18:24,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742351_1527 (size=12301) 2024-11-14T06:18:24,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:24,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731565164517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:24,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:24,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:24,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:24,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:24,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731565164827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:24,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:24,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:24,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:24,868 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/7c0b7a6e46464c7a92e73e0f615a057a 2024-11-14T06:18:24,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/52e1b106661d4d8191675502a693165a is 50, key is test_row_0/C:col10/1731565103135/Put/seqid=0 2024-11-14T06:18:24,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742352_1528 (size=12301) 2024-11-14T06:18:25,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-14T06:18:25,278 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/52e1b106661d4d8191675502a693165a 2024-11-14T06:18:25,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/08ee0adeb2e74f1db18f5a0d451026f7 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/08ee0adeb2e74f1db18f5a0d451026f7 2024-11-14T06:18:25,284 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/08ee0adeb2e74f1db18f5a0d451026f7, entries=150, sequenceid=398, filesize=12.0 K 2024-11-14T06:18:25,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/7c0b7a6e46464c7a92e73e0f615a057a as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/7c0b7a6e46464c7a92e73e0f615a057a 2024-11-14T06:18:25,288 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/7c0b7a6e46464c7a92e73e0f615a057a, entries=150, sequenceid=398, filesize=12.0 K 2024-11-14T06:18:25,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/52e1b106661d4d8191675502a693165a as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/52e1b106661d4d8191675502a693165a 2024-11-14T06:18:25,292 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/52e1b106661d4d8191675502a693165a, entries=150, sequenceid=398, filesize=12.0 K 2024-11-14T06:18:25,293 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for c151951bebfcfd871997dfedda0e3149 in 1301ms, sequenceid=398, compaction requested=false 2024-11-14T06:18:25,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for c151951bebfcfd871997dfedda0e3149: 2024-11-14T06:18:25,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:25,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-14T06:18:25,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-14T06:18:25,297 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-14T06:18:25,297 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2220 sec 2024-11-14T06:18:25,300 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 2.2320 sec 2024-11-14T06:18:25,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on c151951bebfcfd871997dfedda0e3149 2024-11-14T06:18:25,352 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c151951bebfcfd871997dfedda0e3149 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-14T06:18:25,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=A 2024-11-14T06:18:25,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:25,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=B 2024-11-14T06:18:25,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:25,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=C 2024-11-14T06:18:25,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:25,357 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/a0c7771359694e22ba6b10225a2669b2 is 50, key is test_row_0/A:col10/1731565105350/Put/seqid=0 2024-11-14T06:18:25,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742353_1529 (size=14741) 2024-11-14T06:18:25,364 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/a0c7771359694e22ba6b10225a2669b2 2024-11-14T06:18:25,370 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/6cafe902e8944e51bdbaf8136e2ed589 is 50, key is test_row_0/B:col10/1731565105350/Put/seqid=0 2024-11-14T06:18:25,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742354_1530 (size=12301) 2024-11-14T06:18:25,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:25,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47628 deadline: 1731565165410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:25,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:25,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:25,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:25,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:25,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47606 deadline: 1731565165411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:25,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:25,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731565165421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:25,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:25,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:25,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:25,425 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731565105399, max timestamp=1731565105422Num uniques 859; 2024-11-14T06:18:25,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T06:18:25,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731565105399, max timestamp=1731565105422Num uniques 859; 2024-11-14T06:18:25,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T06:18:25,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:25,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:25,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:25,428 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731565105399, max timestamp=1731565105422Num uniques 859; 2024-11-14T06:18:25,428 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T06:18:25,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:25,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47634 deadline: 1731565165533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:25,543 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:25,543 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:25,543 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:25,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:25,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47606 deadline: 1731565165537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:25,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:25,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:25,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:25,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:25,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47628 deadline: 1731565165541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:25,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:25,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:25,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:25,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:25,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731565165542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:25,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:25,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:25,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:25,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:25,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47634 deadline: 1731565165648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:25,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:25,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:25,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:25,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:25,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47628 deadline: 1731565165658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:25,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:25,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:25,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:25,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:25,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731565165667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:25,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:25,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:25,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:25,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:25,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47606 deadline: 1731565165757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:25,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:25,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:25,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:25,777 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/6cafe902e8944e51bdbaf8136e2ed589 2024-11-14T06:18:25,783 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/44364c4c9f924e0bb5a0e922d2b7b373 is 50, key is test_row_0/C:col10/1731565105350/Put/seqid=0 2024-11-14T06:18:25,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742355_1531 (size=12301) 2024-11-14T06:18:25,794 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/44364c4c9f924e0bb5a0e922d2b7b373 2024-11-14T06:18:25,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/a0c7771359694e22ba6b10225a2669b2 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/a0c7771359694e22ba6b10225a2669b2 2024-11-14T06:18:25,801 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/a0c7771359694e22ba6b10225a2669b2, entries=200, sequenceid=412, filesize=14.4 K 2024-11-14T06:18:25,802 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/6cafe902e8944e51bdbaf8136e2ed589 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/6cafe902e8944e51bdbaf8136e2ed589 2024-11-14T06:18:25,805 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/6cafe902e8944e51bdbaf8136e2ed589, entries=150, sequenceid=412, filesize=12.0 K 2024-11-14T06:18:25,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/44364c4c9f924e0bb5a0e922d2b7b373 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/44364c4c9f924e0bb5a0e922d2b7b373 2024-11-14T06:18:25,809 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/44364c4c9f924e0bb5a0e922d2b7b373, entries=150, sequenceid=412, filesize=12.0 K 2024-11-14T06:18:25,811 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for c151951bebfcfd871997dfedda0e3149 in 460ms, sequenceid=412, compaction requested=true 2024-11-14T06:18:25,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c151951bebfcfd871997dfedda0e3149: 2024-11-14T06:18:25,811 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:18:25,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c151951bebfcfd871997dfedda0e3149:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T06:18:25,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:25,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c151951bebfcfd871997dfedda0e3149:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T06:18:25,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:18:25,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c151951bebfcfd871997dfedda0e3149:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T06:18:25,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T06:18:25,812 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:18:25,817 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:18:25,817 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): c151951bebfcfd871997dfedda0e3149/A is initiating minor compaction (all files) 2024-11-14T06:18:25,817 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c151951bebfcfd871997dfedda0e3149/A in TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:25,817 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:18:25,817 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): c151951bebfcfd871997dfedda0e3149/B is initiating minor compaction (all files) 2024-11-14T06:18:25,817 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/6002b7bb708e462c8b1237790e60db13, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/08ee0adeb2e74f1db18f5a0d451026f7, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/a0c7771359694e22ba6b10225a2669b2] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp, totalSize=39.2 K 2024-11-14T06:18:25,817 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c151951bebfcfd871997dfedda0e3149/B in TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:25,817 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/b7c392ff813c494cab23d5306835e3a2, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/7c0b7a6e46464c7a92e73e0f615a057a, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/6cafe902e8944e51bdbaf8136e2ed589] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp, totalSize=36.8 K 2024-11-14T06:18:25,817 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 6002b7bb708e462c8b1237790e60db13, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1731565103035 2024-11-14T06:18:25,817 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7c392ff813c494cab23d5306835e3a2, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1731565103035 2024-11-14T06:18:25,818 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 08ee0adeb2e74f1db18f5a0d451026f7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731565103135 2024-11-14T06:18:25,818 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c0b7a6e46464c7a92e73e0f615a057a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731565103135 2024-11-14T06:18:25,818 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting a0c7771359694e22ba6b10225a2669b2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1731565104144 2024-11-14T06:18:25,818 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6cafe902e8944e51bdbaf8136e2ed589, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1731565104166 2024-11-14T06:18:25,829 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c151951bebfcfd871997dfedda0e3149#B#compaction#456 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:18:25,830 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/4eec431f13154e47a94fe7bdff300fda is 50, key is test_row_0/B:col10/1731565105350/Put/seqid=0 2024-11-14T06:18:25,834 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c151951bebfcfd871997dfedda0e3149#A#compaction#457 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:18:25,835 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/d7c39267acdd4ee98b194459a9e791e4 is 50, key is test_row_0/A:col10/1731565105350/Put/seqid=0 2024-11-14T06:18:25,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742356_1532 (size=13187) 2024-11-14T06:18:25,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742357_1533 (size=13187) 2024-11-14T06:18:25,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on c151951bebfcfd871997dfedda0e3149 2024-11-14T06:18:25,879 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c151951bebfcfd871997dfedda0e3149 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-14T06:18:25,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=A 2024-11-14T06:18:25,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:18:25,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=B 2024-11-14T06:18:25,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:18:25,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=C 2024-11-14T06:18:25,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:18:25,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/0025c3e91cb04d8c998df8202a071f10 is 50, key is test_row_0/A:col10/1731565105540/Put/seqid=0 2024-11-14T06:18:25,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742358_1534 (size=14741) 2024-11-14T06:18:25,908 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/0025c3e91cb04d8c998df8202a071f10 2024-11-14T06:18:25,920 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/3e201955c7fb44b0ac75b7eae5adbf8b is 50, key is test_row_0/B:col10/1731565105540/Put/seqid=0 2024-11-14T06:18:25,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:25,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47628 deadline: 1731565165917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:25,923 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:25,923 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:25,923 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:25,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742359_1535 (size=12301) 2024-11-14T06:18:25,930 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/3e201955c7fb44b0ac75b7eae5adbf8b 2024-11-14T06:18:25,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:25,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731565165919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:25,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:25,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:25,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:25,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:25,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47634 deadline: 1731565165921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:25,939 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/d66c8fb12b554bd6816eb537e87af4ce is 50, key is test_row_0/C:col10/1731565105540/Put/seqid=0 2024-11-14T06:18:25,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:25,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:25,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:25,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742360_1536 (size=12301) 2024-11-14T06:18:26,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:26,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47628 deadline: 1731565166027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:26,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:26,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:26,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:26,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:26,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731565166047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:26,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:26,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47634 deadline: 1731565166047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:26,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:26,059 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:26,059 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:26,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:26,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:26,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:26,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:26,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47606 deadline: 1731565166078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:26,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:26,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:26,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:26,249 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/4eec431f13154e47a94fe7bdff300fda as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/4eec431f13154e47a94fe7bdff300fda 2024-11-14T06:18:26,253 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c151951bebfcfd871997dfedda0e3149/B of c151951bebfcfd871997dfedda0e3149 into 4eec431f13154e47a94fe7bdff300fda(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:18:26,253 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c151951bebfcfd871997dfedda0e3149: 2024-11-14T06:18:26,253 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., storeName=c151951bebfcfd871997dfedda0e3149/B, priority=13, startTime=1731565105811; duration=0sec 2024-11-14T06:18:26,253 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/d7c39267acdd4ee98b194459a9e791e4 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/d7c39267acdd4ee98b194459a9e791e4 2024-11-14T06:18:26,253 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:18:26,253 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c151951bebfcfd871997dfedda0e3149:B 2024-11-14T06:18:26,253 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:18:26,254 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:18:26,254 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): c151951bebfcfd871997dfedda0e3149/C is initiating minor compaction (all files) 2024-11-14T06:18:26,254 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c151951bebfcfd871997dfedda0e3149/C in TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:26,254 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/66dd04b4cdae49748fc5133ffecbea8a, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/52e1b106661d4d8191675502a693165a, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/44364c4c9f924e0bb5a0e922d2b7b373] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp, totalSize=36.8 K 2024-11-14T06:18:26,254 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66dd04b4cdae49748fc5133ffecbea8a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1731565103035 2024-11-14T06:18:26,255 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52e1b106661d4d8191675502a693165a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731565103135 2024-11-14T06:18:26,255 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44364c4c9f924e0bb5a0e922d2b7b373, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1731565104166 2024-11-14T06:18:26,257 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c151951bebfcfd871997dfedda0e3149/A of c151951bebfcfd871997dfedda0e3149 into d7c39267acdd4ee98b194459a9e791e4(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:18:26,257 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c151951bebfcfd871997dfedda0e3149: 2024-11-14T06:18:26,257 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., storeName=c151951bebfcfd871997dfedda0e3149/A, priority=13, startTime=1731565105811; duration=0sec 2024-11-14T06:18:26,257 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:26,257 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c151951bebfcfd871997dfedda0e3149:A 2024-11-14T06:18:26,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:26,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47628 deadline: 1731565166247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:26,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:26,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:26,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:26,264 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c151951bebfcfd871997dfedda0e3149#C#compaction#461 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:18:26,264 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/45a4e6bf6b0d48e7b693ce32e83d7628 is 50, key is test_row_0/C:col10/1731565105350/Put/seqid=0 2024-11-14T06:18:26,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742361_1537 (size=13187) 2024-11-14T06:18:26,271 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/45a4e6bf6b0d48e7b693ce32e83d7628 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/45a4e6bf6b0d48e7b693ce32e83d7628 2024-11-14T06:18:26,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:26,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47634 deadline: 1731565166268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:26,273 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:26,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:26,273 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:26,273 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:26,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731565166269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:26,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:26,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:26,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:26,275 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c151951bebfcfd871997dfedda0e3149/C of c151951bebfcfd871997dfedda0e3149 into 45a4e6bf6b0d48e7b693ce32e83d7628(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:18:26,275 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c151951bebfcfd871997dfedda0e3149: 2024-11-14T06:18:26,275 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., storeName=c151951bebfcfd871997dfedda0e3149/C, priority=13, startTime=1731565105811; duration=0sec 2024-11-14T06:18:26,275 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:26,276 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c151951bebfcfd871997dfedda0e3149:C 2024-11-14T06:18:26,352 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/d66c8fb12b554bd6816eb537e87af4ce 2024-11-14T06:18:26,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/0025c3e91cb04d8c998df8202a071f10 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/0025c3e91cb04d8c998df8202a071f10 2024-11-14T06:18:26,361 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/0025c3e91cb04d8c998df8202a071f10, entries=200, sequenceid=449, filesize=14.4 K 2024-11-14T06:18:26,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/3e201955c7fb44b0ac75b7eae5adbf8b as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/3e201955c7fb44b0ac75b7eae5adbf8b 2024-11-14T06:18:26,364 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/3e201955c7fb44b0ac75b7eae5adbf8b, entries=150, sequenceid=449, filesize=12.0 K 2024-11-14T06:18:26,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/d66c8fb12b554bd6816eb537e87af4ce as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/d66c8fb12b554bd6816eb537e87af4ce 2024-11-14T06:18:26,367 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/d66c8fb12b554bd6816eb537e87af4ce, entries=150, sequenceid=449, filesize=12.0 K 2024-11-14T06:18:26,368 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=53.67 KB/54960 for c151951bebfcfd871997dfedda0e3149 in 489ms, sequenceid=449, compaction requested=false 2024-11-14T06:18:26,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c151951bebfcfd871997dfedda0e3149: 2024-11-14T06:18:26,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on c151951bebfcfd871997dfedda0e3149 2024-11-14T06:18:26,573 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c151951bebfcfd871997dfedda0e3149 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-14T06:18:26,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=A 2024-11-14T06:18:26,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:26,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=B 2024-11-14T06:18:26,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:26,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=C 2024-11-14T06:18:26,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:26,576 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/e49d499ee2d14c6abac0feb6bcbbcae2 is 50, key is test_row_0/A:col10/1731565105918/Put/seqid=0 2024-11-14T06:18:26,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742362_1538 (size=14741) 2024-11-14T06:18:26,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:26,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47628 deadline: 1731565166608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:26,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:26,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:26,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:26,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:26,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47634 deadline: 1731565166611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:26,618 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:26,618 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:26,618 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:26,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:26,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731565166617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:26,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:26,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:26,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:26,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47606 deadline: 1731565166618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:26,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:26,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:26,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:26,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:26,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:26,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47628 deadline: 1731565166727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:26,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:26,734 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:26,734 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:26,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47634 deadline: 1731565166727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:26,734 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:26,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:26,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:26,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:26,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731565166727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:26,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:26,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:26,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:26,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:26,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47606 deadline: 1731565166727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:26,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:26,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:26,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:26,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:26,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:26,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47628 deadline: 1731565166937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:26,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:26,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47634 deadline: 1731565166937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:26,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:26,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:26,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:26,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:26,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731565166937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:26,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:26,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:26,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:26,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:26,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:26,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:26,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:26,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47606 deadline: 1731565166937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:26,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:26,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:26,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:26,986 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/e49d499ee2d14c6abac0feb6bcbbcae2 2024-11-14T06:18:26,991 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/307f9eb0e6ef4dc4953d1abb410289d6 is 50, key is test_row_0/B:col10/1731565105918/Put/seqid=0 2024-11-14T06:18:26,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742363_1539 (size=12301) 2024-11-14T06:18:26,996 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/307f9eb0e6ef4dc4953d1abb410289d6 2024-11-14T06:18:27,001 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/e4ad394b98c74d0d8be8b9fade1fdc68 is 50, key is test_row_0/C:col10/1731565105918/Put/seqid=0 2024-11-14T06:18:27,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742364_1540 (size=12301) 2024-11-14T06:18:27,014 INFO [Thread-2043 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2043. 2024-11-14T06:18:27,014 DEBUG [Thread-2043 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T06:18:27,014 DEBUG [Thread-2043 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:27,014 DEBUG [Thread-2043 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:27,014 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:18:27,015 INFO [Thread-2037 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2037. 2024-11-14T06:18:27,015 DEBUG [Thread-2037 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T06:18:27,015 DEBUG [Thread-2037 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:27,015 DEBUG [Thread-2037 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:27,015 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:18:27,018 INFO [Thread-2045 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2045. 2024-11-14T06:18:27,018 DEBUG [Thread-2045 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T06:18:27,018 DEBUG [Thread-2045 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:27,018 DEBUG [Thread-2045 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:27,018 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:18:27,024 INFO [Thread-2039 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2039. 2024-11-14T06:18:27,024 DEBUG [Thread-2039 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T06:18:27,024 DEBUG [Thread-2039 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:27,024 DEBUG [Thread-2039 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:27,024 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:18:27,026 INFO [Thread-2041 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2041. 2024-11-14T06:18:27,026 DEBUG [Thread-2041 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T06:18:27,026 DEBUG [Thread-2041 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:27,026 DEBUG [Thread-2041 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:27,026 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:18:27,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-14T06:18:27,238 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:18:27,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:27,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:27,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:27,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47634 deadline: 1731565167247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:27,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47628 deadline: 1731565167247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:27,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731565167247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:27,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:27,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:27,247 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:27,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:27,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47606 deadline: 1731565167247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:27,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:27,247 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:27,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:27,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:27,247 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:27,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:27,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 , the old value is region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:27,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:27,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=2 because the exception is null or not the one we care about 2024-11-14T06:18:27,405 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/e4ad394b98c74d0d8be8b9fade1fdc68 2024-11-14T06:18:27,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/e49d499ee2d14c6abac0feb6bcbbcae2 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/e49d499ee2d14c6abac0feb6bcbbcae2 2024-11-14T06:18:27,412 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/e49d499ee2d14c6abac0feb6bcbbcae2, entries=200, sequenceid=464, filesize=14.4 K 2024-11-14T06:18:27,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/307f9eb0e6ef4dc4953d1abb410289d6 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/307f9eb0e6ef4dc4953d1abb410289d6 2024-11-14T06:18:27,416 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/307f9eb0e6ef4dc4953d1abb410289d6, entries=150, sequenceid=464, filesize=12.0 K 2024-11-14T06:18:27,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/e4ad394b98c74d0d8be8b9fade1fdc68 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/e4ad394b98c74d0d8be8b9fade1fdc68 2024-11-14T06:18:27,419 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/e4ad394b98c74d0d8be8b9fade1fdc68, entries=150, sequenceid=464, filesize=12.0 K 2024-11-14T06:18:27,420 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for c151951bebfcfd871997dfedda0e3149 in 847ms, sequenceid=464, compaction requested=true 2024-11-14T06:18:27,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c151951bebfcfd871997dfedda0e3149: 2024-11-14T06:18:27,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c151951bebfcfd871997dfedda0e3149:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T06:18:27,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:27,420 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:18:27,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c151951bebfcfd871997dfedda0e3149:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T06:18:27,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:27,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c151951bebfcfd871997dfedda0e3149:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T06:18:27,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:18:27,420 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:18:27,421 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:18:27,421 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42669 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:18:27,421 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): c151951bebfcfd871997dfedda0e3149/B is initiating minor compaction (all files) 2024-11-14T06:18:27,421 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): c151951bebfcfd871997dfedda0e3149/A is initiating minor compaction (all files) 2024-11-14T06:18:27,421 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c151951bebfcfd871997dfedda0e3149/A in TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:27,421 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c151951bebfcfd871997dfedda0e3149/B in TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:27,422 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/d7c39267acdd4ee98b194459a9e791e4, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/0025c3e91cb04d8c998df8202a071f10, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/e49d499ee2d14c6abac0feb6bcbbcae2] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp, totalSize=41.7 K 2024-11-14T06:18:27,422 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/4eec431f13154e47a94fe7bdff300fda, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/3e201955c7fb44b0ac75b7eae5adbf8b, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/307f9eb0e6ef4dc4953d1abb410289d6] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp, totalSize=36.9 K 2024-11-14T06:18:27,422 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7c39267acdd4ee98b194459a9e791e4, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1731565104166 2024-11-14T06:18:27,422 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 4eec431f13154e47a94fe7bdff300fda, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1731565104166 2024-11-14T06:18:27,422 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0025c3e91cb04d8c998df8202a071f10, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1731565105518 2024-11-14T06:18:27,422 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e201955c7fb44b0ac75b7eae5adbf8b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1731565105518 2024-11-14T06:18:27,422 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting e49d499ee2d14c6abac0feb6bcbbcae2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=464, earliestPutTs=1731565105897 2024-11-14T06:18:27,422 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 307f9eb0e6ef4dc4953d1abb410289d6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=464, earliestPutTs=1731565105897 2024-11-14T06:18:27,427 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c151951bebfcfd871997dfedda0e3149#B#compaction#465 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:18:27,428 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/c334f7826890482facdfdbacfa809d79 is 50, key is test_row_0/B:col10/1731565105918/Put/seqid=0 2024-11-14T06:18:27,428 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c151951bebfcfd871997dfedda0e3149#A#compaction#466 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:18:27,429 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/b55c0444a31040a2a1396578c6e09e5d is 50, key is test_row_0/A:col10/1731565105918/Put/seqid=0 2024-11-14T06:18:27,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742365_1541 (size=13289) 2024-11-14T06:18:27,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742366_1542 (size=13289) 2024-11-14T06:18:27,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on c151951bebfcfd871997dfedda0e3149 2024-11-14T06:18:27,759 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c151951bebfcfd871997dfedda0e3149 3/3 column families, dataSize=134.18 KB heapSize=399.89 KB 2024-11-14T06:18:27,759 INFO [Thread-2032 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2032. 2024-11-14T06:18:27,759 DEBUG [Thread-2032 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T06:18:27,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=A 2024-11-14T06:18:27,759 DEBUG [Thread-2032 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:27,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:18:27,759 INFO [Thread-2030 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2030. 2024-11-14T06:18:27,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=B 2024-11-14T06:18:27,759 DEBUG [Thread-2032 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:27,759 DEBUG [Thread-2030 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T06:18:27,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:18:27,759 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:18:27,759 DEBUG [Thread-2030 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:27,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=C 2024-11-14T06:18:27,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:18:27,759 DEBUG [Thread-2030 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:27,760 INFO [Thread-2028 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2028. 2024-11-14T06:18:27,760 DEBUG [Thread-2028 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T06:18:27,760 DEBUG [Thread-2028 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:27,760 DEBUG [Thread-2028 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:27,760 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:18:27,761 INFO [Thread-2026 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2026. 2024-11-14T06:18:27,761 DEBUG [Thread-2026 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T06:18:27,761 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:18:27,761 DEBUG [Thread-2026 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:27,761 DEBUG [Thread-2026 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:27,762 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:18:27,767 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/315e0aca3a40410db884913a02d77f50 is 50, key is test_row_0/A:col10/1731565106605/Put/seqid=0 2024-11-14T06:18:27,769 INFO [Thread-2034 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2034. 2024-11-14T06:18:27,769 DEBUG [Thread-2034 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T06:18:27,769 DEBUG [Thread-2034 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:27,769 DEBUG [Thread-2034 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:27,769 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:18:27,769 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T06:18:27,769 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 100 2024-11-14T06:18:27,770 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 111 2024-11-14T06:18:27,770 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 78 2024-11-14T06:18:27,770 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 93 2024-11-14T06:18:27,770 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 16 2024-11-14T06:18:27,770 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T06:18:27,770 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T06:18:27,770 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1134 2024-11-14T06:18:27,770 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3402 rows 2024-11-14T06:18:27,770 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1128 2024-11-14T06:18:27,770 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3383 rows 2024-11-14T06:18:27,770 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1132 2024-11-14T06:18:27,770 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3392 rows 2024-11-14T06:18:27,770 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1146 2024-11-14T06:18:27,770 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3428 rows 2024-11-14T06:18:27,770 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1132 2024-11-14T06:18:27,770 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3395 rows 2024-11-14T06:18:27,770 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T06:18:27,770 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T06:18:27,770 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:27,770 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:27,770 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:18:27,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742367_1543 (size=12301) 2024-11-14T06:18:27,771 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-14T06:18:27,771 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=487 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/315e0aca3a40410db884913a02d77f50 2024-11-14T06:18:27,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T06:18:27,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-14T06:18:27,774 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731565107774"}]},"ts":"1731565107774"} 2024-11-14T06:18:27,775 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T06:18:27,775 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T06:18:27,776 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T06:18:27,778 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c151951bebfcfd871997dfedda0e3149, UNASSIGN}] 2024-11-14T06:18:27,779 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c151951bebfcfd871997dfedda0e3149, UNASSIGN 2024-11-14T06:18:27,780 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=c151951bebfcfd871997dfedda0e3149, regionState=CLOSING, regionLocation=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:27,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/26e5f3dc578841368e7f5715782c3f83 is 50, key is test_row_0/B:col10/1731565106605/Put/seqid=0 2024-11-14T06:18:27,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742368_1544 (size=12301) 2024-11-14T06:18:27,788 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c151951bebfcfd871997dfedda0e3149, UNASSIGN because future has completed 2024-11-14T06:18:27,788 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T06:18:27,788 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=487 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/26e5f3dc578841368e7f5715782c3f83 2024-11-14T06:18:27,788 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945}] 2024-11-14T06:18:27,793 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/12245e7a8cd2483c822ff1825e97f92c is 50, key is test_row_0/C:col10/1731565106605/Put/seqid=0 2024-11-14T06:18:27,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742369_1545 (size=12301) 2024-11-14T06:18:27,839 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/c334f7826890482facdfdbacfa809d79 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/c334f7826890482facdfdbacfa809d79 2024-11-14T06:18:27,842 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/b55c0444a31040a2a1396578c6e09e5d as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/b55c0444a31040a2a1396578c6e09e5d 2024-11-14T06:18:27,844 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c151951bebfcfd871997dfedda0e3149/B of c151951bebfcfd871997dfedda0e3149 into c334f7826890482facdfdbacfa809d79(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:18:27,844 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c151951bebfcfd871997dfedda0e3149: 2024-11-14T06:18:27,844 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., storeName=c151951bebfcfd871997dfedda0e3149/B, priority=13, startTime=1731565107420; duration=0sec 2024-11-14T06:18:27,844 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:18:27,844 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c151951bebfcfd871997dfedda0e3149:B 2024-11-14T06:18:27,844 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:18:27,846 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c151951bebfcfd871997dfedda0e3149/A of c151951bebfcfd871997dfedda0e3149 into b55c0444a31040a2a1396578c6e09e5d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:18:27,846 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c151951bebfcfd871997dfedda0e3149: 2024-11-14T06:18:27,846 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., storeName=c151951bebfcfd871997dfedda0e3149/A, priority=13, startTime=1731565107420; duration=0sec 2024-11-14T06:18:27,846 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:27,846 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c151951bebfcfd871997dfedda0e3149:A 2024-11-14T06:18:27,848 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:18:27,848 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): c151951bebfcfd871997dfedda0e3149/C is initiating minor compaction (all files) 2024-11-14T06:18:27,849 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c151951bebfcfd871997dfedda0e3149/C in TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:27,849 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/45a4e6bf6b0d48e7b693ce32e83d7628, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/d66c8fb12b554bd6816eb537e87af4ce, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/e4ad394b98c74d0d8be8b9fade1fdc68] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp, totalSize=36.9 K 2024-11-14T06:18:27,849 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 45a4e6bf6b0d48e7b693ce32e83d7628, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1731565104166 2024-11-14T06:18:27,849 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting d66c8fb12b554bd6816eb537e87af4ce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1731565105518 2024-11-14T06:18:27,849 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting e4ad394b98c74d0d8be8b9fade1fdc68, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=464, earliestPutTs=1731565105897 2024-11-14T06:18:27,854 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c151951bebfcfd871997dfedda0e3149#C#compaction#470 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:18:27,855 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/d09d82a546e24bebb892ebb72b09e1b9 is 50, key is test_row_0/C:col10/1731565105918/Put/seqid=0 2024-11-14T06:18:27,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742370_1546 (size=13289) 2024-11-14T06:18:27,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-14T06:18:27,941 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(122): Close c151951bebfcfd871997dfedda0e3149 2024-11-14T06:18:27,941 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T06:18:27,941 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1722): Closing c151951bebfcfd871997dfedda0e3149, disabling compactions & flushes 2024-11-14T06:18:27,941 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1993): waiting for 1 compactions & cache flush to complete for region TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:28,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-14T06:18:28,196 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=487 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/12245e7a8cd2483c822ff1825e97f92c 2024-11-14T06:18:28,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/315e0aca3a40410db884913a02d77f50 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/315e0aca3a40410db884913a02d77f50 2024-11-14T06:18:28,202 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/315e0aca3a40410db884913a02d77f50, entries=150, sequenceid=487, filesize=12.0 K 2024-11-14T06:18:28,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/26e5f3dc578841368e7f5715782c3f83 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/26e5f3dc578841368e7f5715782c3f83 2024-11-14T06:18:28,204 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/26e5f3dc578841368e7f5715782c3f83, entries=150, sequenceid=487, filesize=12.0 K 2024-11-14T06:18:28,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/12245e7a8cd2483c822ff1825e97f92c as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/12245e7a8cd2483c822ff1825e97f92c 2024-11-14T06:18:28,207 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/12245e7a8cd2483c822ff1825e97f92c, entries=150, sequenceid=487, filesize=12.0 K 2024-11-14T06:18:28,207 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~399.84 KB/409440, currentSize=13.42 KB/13740 for c151951bebfcfd871997dfedda0e3149 in 448ms, sequenceid=487, compaction requested=false 2024-11-14T06:18:28,207 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c151951bebfcfd871997dfedda0e3149: 2024-11-14T06:18:28,208 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1993): waiting for 1 compactions to complete for region TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:28,263 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/d09d82a546e24bebb892ebb72b09e1b9 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/d09d82a546e24bebb892ebb72b09e1b9 2024-11-14T06:18:28,267 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c151951bebfcfd871997dfedda0e3149/C of c151951bebfcfd871997dfedda0e3149 into d09d82a546e24bebb892ebb72b09e1b9(size=13.0 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:18:28,267 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c151951bebfcfd871997dfedda0e3149: 2024-11-14T06:18:28,267 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., storeName=c151951bebfcfd871997dfedda0e3149/C, priority=13, startTime=1731565107420; duration=0sec 2024-11-14T06:18:28,267 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:28,267 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:28,267 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. after waiting 0 ms 2024-11-14T06:18:28,267 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:28,267 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:28,267 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c151951bebfcfd871997dfedda0e3149:C 2024-11-14T06:18:28,267 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(2902): Flushing c151951bebfcfd871997dfedda0e3149 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-14T06:18:28,267 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=A 2024-11-14T06:18:28,267 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:28,267 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=B 2024-11-14T06:18:28,267 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:28,267 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c151951bebfcfd871997dfedda0e3149, store=C 2024-11-14T06:18:28,267 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:28,273 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/ca9b102709504cdca55a713315ff658c is 50, key is test_row_0/A:col10/1731565107760/Put/seqid=0 2024-11-14T06:18:28,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742371_1547 (size=9857) 2024-11-14T06:18:28,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-14T06:18:28,678 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=495 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/ca9b102709504cdca55a713315ff658c 2024-11-14T06:18:28,688 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/9378e3194df6483084ea9e9f82e009ff is 50, key is test_row_0/B:col10/1731565107760/Put/seqid=0 2024-11-14T06:18:28,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742372_1548 (size=9857) 2024-11-14T06:18:28,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-14T06:18:29,091 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=495 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/9378e3194df6483084ea9e9f82e009ff 2024-11-14T06:18:29,096 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/a6e97d9b570840e38cc932ae8807b90e is 50, key is test_row_0/C:col10/1731565107760/Put/seqid=0 2024-11-14T06:18:29,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742373_1549 (size=9857) 2024-11-14T06:18:29,500 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=495 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/a6e97d9b570840e38cc932ae8807b90e 2024-11-14T06:18:29,503 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/A/ca9b102709504cdca55a713315ff658c as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/ca9b102709504cdca55a713315ff658c 2024-11-14T06:18:29,506 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/ca9b102709504cdca55a713315ff658c, entries=100, sequenceid=495, filesize=9.6 K 2024-11-14T06:18:29,507 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/B/9378e3194df6483084ea9e9f82e009ff as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/9378e3194df6483084ea9e9f82e009ff 2024-11-14T06:18:29,509 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/9378e3194df6483084ea9e9f82e009ff, entries=100, sequenceid=495, filesize=9.6 K 2024-11-14T06:18:29,510 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/.tmp/C/a6e97d9b570840e38cc932ae8807b90e as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/a6e97d9b570840e38cc932ae8807b90e 2024-11-14T06:18:29,513 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/a6e97d9b570840e38cc932ae8807b90e, entries=100, sequenceid=495, filesize=9.6 K 2024-11-14T06:18:29,513 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for c151951bebfcfd871997dfedda0e3149 in 1246ms, sequenceid=495, compaction requested=true 2024-11-14T06:18:29,514 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/55d4550cc1f44f80b21e9b4d3ec95ff5, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/7c98c295bb254185980907b085237c01, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/c645e8861fa9477382e22094daebe1ed, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/07e0dcf2b6554228a15be359ac41a892, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/3fdf416388f048fd96526ddead040844, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/5cbd618ef7804b8b90894e7009100d91, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/de4d79515e584fc4a70c3465d9559296, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/70b723c1355f440b8e7de2269758d7f6, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/a725821d5a434587932956d29d135c86, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/f2bc4974354f4c6caa26525d334df4ed, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/5b48a8b71e314300a7c7df412c365e43, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/07aff5d543a54275a83eb76f4b2378f6, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/89262aa98ef54e3bbe607b97f7aac656, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/3b89aa76c12a4240ab018e36f3aabdcb, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/32b364ab38354cf5b0d3e96418f9cd04, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/7c6fde6805434f678e1711a26ee1f81d, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/ce2c4cf6cf9e4663916e86ceb5308c04, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/2217513902b34c218d919ad0596f7527, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/f09e3a263161483f9f15a555d3e1151c, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/0dd4c9f1aad14d63a9f70ddb6f8bfd15, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/93f445866d1144ea85efcbcdbcb8a22d, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/38f700fc50784855b30a356c471dbfa8, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/b3e063d5a2124f7199d60c048bc03bbf, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/6002b7bb708e462c8b1237790e60db13, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/08ee0adeb2e74f1db18f5a0d451026f7, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/a0c7771359694e22ba6b10225a2669b2, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/d7c39267acdd4ee98b194459a9e791e4, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/0025c3e91cb04d8c998df8202a071f10, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/e49d499ee2d14c6abac0feb6bcbbcae2] to archive 2024-11-14T06:18:29,514 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T06:18:29,516 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/55d4550cc1f44f80b21e9b4d3ec95ff5 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/55d4550cc1f44f80b21e9b4d3ec95ff5 2024-11-14T06:18:29,516 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/7c98c295bb254185980907b085237c01 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/7c98c295bb254185980907b085237c01 2024-11-14T06:18:29,517 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/c645e8861fa9477382e22094daebe1ed to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/c645e8861fa9477382e22094daebe1ed 2024-11-14T06:18:29,518 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/07e0dcf2b6554228a15be359ac41a892 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/07e0dcf2b6554228a15be359ac41a892 2024-11-14T06:18:29,519 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/3fdf416388f048fd96526ddead040844 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/3fdf416388f048fd96526ddead040844 2024-11-14T06:18:29,519 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/5cbd618ef7804b8b90894e7009100d91 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/5cbd618ef7804b8b90894e7009100d91 2024-11-14T06:18:29,520 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/de4d79515e584fc4a70c3465d9559296 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/de4d79515e584fc4a70c3465d9559296 2024-11-14T06:18:29,521 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/70b723c1355f440b8e7de2269758d7f6 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/70b723c1355f440b8e7de2269758d7f6 2024-11-14T06:18:29,522 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/a725821d5a434587932956d29d135c86 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/a725821d5a434587932956d29d135c86 2024-11-14T06:18:29,522 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/f2bc4974354f4c6caa26525d334df4ed to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/f2bc4974354f4c6caa26525d334df4ed 2024-11-14T06:18:29,523 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/5b48a8b71e314300a7c7df412c365e43 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/5b48a8b71e314300a7c7df412c365e43 2024-11-14T06:18:29,524 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/07aff5d543a54275a83eb76f4b2378f6 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/07aff5d543a54275a83eb76f4b2378f6 2024-11-14T06:18:29,525 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/89262aa98ef54e3bbe607b97f7aac656 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/89262aa98ef54e3bbe607b97f7aac656 2024-11-14T06:18:29,525 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/3b89aa76c12a4240ab018e36f3aabdcb to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/3b89aa76c12a4240ab018e36f3aabdcb 2024-11-14T06:18:29,526 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/32b364ab38354cf5b0d3e96418f9cd04 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/32b364ab38354cf5b0d3e96418f9cd04 2024-11-14T06:18:29,527 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/7c6fde6805434f678e1711a26ee1f81d to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/7c6fde6805434f678e1711a26ee1f81d 2024-11-14T06:18:29,528 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/ce2c4cf6cf9e4663916e86ceb5308c04 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/ce2c4cf6cf9e4663916e86ceb5308c04 2024-11-14T06:18:29,528 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/2217513902b34c218d919ad0596f7527 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/2217513902b34c218d919ad0596f7527 2024-11-14T06:18:29,529 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/f09e3a263161483f9f15a555d3e1151c to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/f09e3a263161483f9f15a555d3e1151c 2024-11-14T06:18:29,530 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/0dd4c9f1aad14d63a9f70ddb6f8bfd15 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/0dd4c9f1aad14d63a9f70ddb6f8bfd15 2024-11-14T06:18:29,531 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/93f445866d1144ea85efcbcdbcb8a22d to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/93f445866d1144ea85efcbcdbcb8a22d 2024-11-14T06:18:29,531 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/38f700fc50784855b30a356c471dbfa8 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/38f700fc50784855b30a356c471dbfa8 2024-11-14T06:18:29,532 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/b3e063d5a2124f7199d60c048bc03bbf to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/b3e063d5a2124f7199d60c048bc03bbf 2024-11-14T06:18:29,533 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/6002b7bb708e462c8b1237790e60db13 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/6002b7bb708e462c8b1237790e60db13 2024-11-14T06:18:29,534 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/08ee0adeb2e74f1db18f5a0d451026f7 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/08ee0adeb2e74f1db18f5a0d451026f7 2024-11-14T06:18:29,535 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/a0c7771359694e22ba6b10225a2669b2 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/a0c7771359694e22ba6b10225a2669b2 2024-11-14T06:18:29,535 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/d7c39267acdd4ee98b194459a9e791e4 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/d7c39267acdd4ee98b194459a9e791e4 2024-11-14T06:18:29,536 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/0025c3e91cb04d8c998df8202a071f10 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/0025c3e91cb04d8c998df8202a071f10 2024-11-14T06:18:29,537 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/e49d499ee2d14c6abac0feb6bcbbcae2 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/e49d499ee2d14c6abac0feb6bcbbcae2 2024-11-14T06:18:29,539 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/1e4629de011c4997985e7e9c6f949ded, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/bcd52a32bc0f4dcc9447b9594c19f5af, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/2244ceab619c4cbfa02adbe698896887, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/e2fc54c3c3f241209f717f2f77e4641e, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/73384b33e24e487bb7be804ad03a80ce, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/ddeecb0e35784d3ca154ae5407f43836, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/6c50e7d4fc85433b854cf578457cb0fc, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/0ae6c5d273964df693dcdf20120fe48c, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/079a007e90404f65bd7a46b5742719ed, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/383b634c434f4d10bb2d6a2709ab7fa1, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/9254b63b509a47b7b897cdbd2ac270e3, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/cbdf47e6151b4a488be97aa0873c2ec1, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/77bd2c24b59e43cea71383ae2dacc68f, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/cc850ec550774274828eef401871b139, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/da20c0982a1c4ef9b891393d52ebfa64, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/03971a5355044dd3bf09732d258b91a9, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/94e40894ba4a430691f23183c028d1c9, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/d8d6c2691f274da5818ac466dc4de369, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/755090207794465686b0ef183a8bb05f, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/9a39cdd4016c43198ad0798516026b36, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/5649e1ca1bd2495cbb427045dc20f48b, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/d9239c85eac24f57a8fc6c391532b8b2, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/b7c392ff813c494cab23d5306835e3a2, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/69d8fb2084134abbbfb7422291c874f6, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/7c0b7a6e46464c7a92e73e0f615a057a, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/4eec431f13154e47a94fe7bdff300fda, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/6cafe902e8944e51bdbaf8136e2ed589, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/3e201955c7fb44b0ac75b7eae5adbf8b, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/307f9eb0e6ef4dc4953d1abb410289d6] to archive 2024-11-14T06:18:29,539 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T06:18:29,541 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/1e4629de011c4997985e7e9c6f949ded to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/1e4629de011c4997985e7e9c6f949ded 2024-11-14T06:18:29,541 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/bcd52a32bc0f4dcc9447b9594c19f5af to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/bcd52a32bc0f4dcc9447b9594c19f5af 2024-11-14T06:18:29,542 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/2244ceab619c4cbfa02adbe698896887 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/2244ceab619c4cbfa02adbe698896887 2024-11-14T06:18:29,543 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/e2fc54c3c3f241209f717f2f77e4641e to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/e2fc54c3c3f241209f717f2f77e4641e 2024-11-14T06:18:29,544 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/73384b33e24e487bb7be804ad03a80ce to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/73384b33e24e487bb7be804ad03a80ce 2024-11-14T06:18:29,545 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/ddeecb0e35784d3ca154ae5407f43836 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/ddeecb0e35784d3ca154ae5407f43836 2024-11-14T06:18:29,545 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/6c50e7d4fc85433b854cf578457cb0fc to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/6c50e7d4fc85433b854cf578457cb0fc 2024-11-14T06:18:29,546 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/0ae6c5d273964df693dcdf20120fe48c to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/0ae6c5d273964df693dcdf20120fe48c 2024-11-14T06:18:29,547 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/079a007e90404f65bd7a46b5742719ed to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/079a007e90404f65bd7a46b5742719ed 2024-11-14T06:18:29,548 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/383b634c434f4d10bb2d6a2709ab7fa1 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/383b634c434f4d10bb2d6a2709ab7fa1 2024-11-14T06:18:29,549 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/9254b63b509a47b7b897cdbd2ac270e3 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/9254b63b509a47b7b897cdbd2ac270e3 2024-11-14T06:18:29,550 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/cbdf47e6151b4a488be97aa0873c2ec1 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/cbdf47e6151b4a488be97aa0873c2ec1 2024-11-14T06:18:29,550 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/77bd2c24b59e43cea71383ae2dacc68f to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/77bd2c24b59e43cea71383ae2dacc68f 2024-11-14T06:18:29,551 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/cc850ec550774274828eef401871b139 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/cc850ec550774274828eef401871b139 2024-11-14T06:18:29,552 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/da20c0982a1c4ef9b891393d52ebfa64 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/da20c0982a1c4ef9b891393d52ebfa64 2024-11-14T06:18:29,553 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/03971a5355044dd3bf09732d258b91a9 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/03971a5355044dd3bf09732d258b91a9 2024-11-14T06:18:29,554 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/94e40894ba4a430691f23183c028d1c9 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/94e40894ba4a430691f23183c028d1c9 2024-11-14T06:18:29,554 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/d8d6c2691f274da5818ac466dc4de369 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/d8d6c2691f274da5818ac466dc4de369 2024-11-14T06:18:29,555 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/755090207794465686b0ef183a8bb05f to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/755090207794465686b0ef183a8bb05f 2024-11-14T06:18:29,556 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/9a39cdd4016c43198ad0798516026b36 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/9a39cdd4016c43198ad0798516026b36 2024-11-14T06:18:29,557 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/5649e1ca1bd2495cbb427045dc20f48b to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/5649e1ca1bd2495cbb427045dc20f48b 2024-11-14T06:18:29,557 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/d9239c85eac24f57a8fc6c391532b8b2 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/d9239c85eac24f57a8fc6c391532b8b2 2024-11-14T06:18:29,558 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/b7c392ff813c494cab23d5306835e3a2 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/b7c392ff813c494cab23d5306835e3a2 2024-11-14T06:18:29,558 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/69d8fb2084134abbbfb7422291c874f6 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/69d8fb2084134abbbfb7422291c874f6 2024-11-14T06:18:29,559 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/7c0b7a6e46464c7a92e73e0f615a057a to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/7c0b7a6e46464c7a92e73e0f615a057a 2024-11-14T06:18:29,560 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/4eec431f13154e47a94fe7bdff300fda to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/4eec431f13154e47a94fe7bdff300fda 2024-11-14T06:18:29,560 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/6cafe902e8944e51bdbaf8136e2ed589 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/6cafe902e8944e51bdbaf8136e2ed589 2024-11-14T06:18:29,561 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/3e201955c7fb44b0ac75b7eae5adbf8b to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/3e201955c7fb44b0ac75b7eae5adbf8b 2024-11-14T06:18:29,561 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/307f9eb0e6ef4dc4953d1abb410289d6 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/307f9eb0e6ef4dc4953d1abb410289d6 2024-11-14T06:18:29,563 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/bec89b3e6de94c6f80f296728adea529, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/b17fd7006793483c8314f1dcec1a0332, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/e267e2a77d7b4879be77129380c4c09f, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/6676e93a2d1045cfa2e4b18c305b10ea, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/cb3301a633ff4e4bb18c27e73529bbd1, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/5e55c53998d047ef9c974103fa07108a, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/bcc49c6cbc424263871d55cf751f57b6, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/576916c40e214785b18324a8bd3c28ac, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/0df943bf6c5c4228a6d6aee2169ae0e2, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/d9a086ed6ef9415d8eb9f205550149a4, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/6a5ed1c0d6e74c819500767755b7dbf0, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/0af4712533054a83b642ec6e8e3a6cb3, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/be3f5aef906349cc8597b0e0c015880e, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/6501d0fec7c147d989220a89f0755220, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/fb31af59de214c51a072c302f1ef8ef6, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/404a54494afd4084bfed1ca2dac7a83a, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/5aa2005443de4ce9aafd5620dc757f7e, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/b4ecd4625d4e4c7e85f535d7e221aa86, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/6c236b2a083f41d2998bd7d5090ddc88, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/0e8b15b1207646fda27aefdde022cfbd, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/5f443cd7ac074339849dd7f1d0aa476b, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/02fac31d63f64f77a065a711ff106ad7, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/66dd04b4cdae49748fc5133ffecbea8a, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/45c2c0c746714f1e8791d00ce906312e, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/52e1b106661d4d8191675502a693165a, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/45a4e6bf6b0d48e7b693ce32e83d7628, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/44364c4c9f924e0bb5a0e922d2b7b373, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/d66c8fb12b554bd6816eb537e87af4ce, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/e4ad394b98c74d0d8be8b9fade1fdc68] to archive 2024-11-14T06:18:29,563 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T06:18:29,565 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/bec89b3e6de94c6f80f296728adea529 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/bec89b3e6de94c6f80f296728adea529 2024-11-14T06:18:29,565 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/b17fd7006793483c8314f1dcec1a0332 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/b17fd7006793483c8314f1dcec1a0332 2024-11-14T06:18:29,566 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/e267e2a77d7b4879be77129380c4c09f to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/e267e2a77d7b4879be77129380c4c09f 2024-11-14T06:18:29,567 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/6676e93a2d1045cfa2e4b18c305b10ea to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/6676e93a2d1045cfa2e4b18c305b10ea 2024-11-14T06:18:29,568 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/cb3301a633ff4e4bb18c27e73529bbd1 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/cb3301a633ff4e4bb18c27e73529bbd1 2024-11-14T06:18:29,568 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/5e55c53998d047ef9c974103fa07108a to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/5e55c53998d047ef9c974103fa07108a 2024-11-14T06:18:29,569 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/bcc49c6cbc424263871d55cf751f57b6 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/bcc49c6cbc424263871d55cf751f57b6 2024-11-14T06:18:29,570 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/576916c40e214785b18324a8bd3c28ac to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/576916c40e214785b18324a8bd3c28ac 2024-11-14T06:18:29,571 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/0df943bf6c5c4228a6d6aee2169ae0e2 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/0df943bf6c5c4228a6d6aee2169ae0e2 2024-11-14T06:18:29,571 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/d9a086ed6ef9415d8eb9f205550149a4 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/d9a086ed6ef9415d8eb9f205550149a4 2024-11-14T06:18:29,572 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/6a5ed1c0d6e74c819500767755b7dbf0 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/6a5ed1c0d6e74c819500767755b7dbf0 2024-11-14T06:18:29,573 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/0af4712533054a83b642ec6e8e3a6cb3 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/0af4712533054a83b642ec6e8e3a6cb3 2024-11-14T06:18:29,574 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/be3f5aef906349cc8597b0e0c015880e to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/be3f5aef906349cc8597b0e0c015880e 2024-11-14T06:18:29,575 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/6501d0fec7c147d989220a89f0755220 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/6501d0fec7c147d989220a89f0755220 2024-11-14T06:18:29,576 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/fb31af59de214c51a072c302f1ef8ef6 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/fb31af59de214c51a072c302f1ef8ef6 2024-11-14T06:18:29,576 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/404a54494afd4084bfed1ca2dac7a83a to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/404a54494afd4084bfed1ca2dac7a83a 2024-11-14T06:18:29,577 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/5aa2005443de4ce9aafd5620dc757f7e to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/5aa2005443de4ce9aafd5620dc757f7e 2024-11-14T06:18:29,578 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/b4ecd4625d4e4c7e85f535d7e221aa86 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/b4ecd4625d4e4c7e85f535d7e221aa86 2024-11-14T06:18:29,579 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/6c236b2a083f41d2998bd7d5090ddc88 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/6c236b2a083f41d2998bd7d5090ddc88 2024-11-14T06:18:29,580 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/0e8b15b1207646fda27aefdde022cfbd to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/0e8b15b1207646fda27aefdde022cfbd 2024-11-14T06:18:29,581 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/5f443cd7ac074339849dd7f1d0aa476b to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/5f443cd7ac074339849dd7f1d0aa476b 2024-11-14T06:18:29,582 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/02fac31d63f64f77a065a711ff106ad7 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/02fac31d63f64f77a065a711ff106ad7 2024-11-14T06:18:29,583 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/66dd04b4cdae49748fc5133ffecbea8a to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/66dd04b4cdae49748fc5133ffecbea8a 2024-11-14T06:18:29,584 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/45c2c0c746714f1e8791d00ce906312e to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/45c2c0c746714f1e8791d00ce906312e 2024-11-14T06:18:29,586 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/52e1b106661d4d8191675502a693165a to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/52e1b106661d4d8191675502a693165a 2024-11-14T06:18:29,587 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/45a4e6bf6b0d48e7b693ce32e83d7628 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/45a4e6bf6b0d48e7b693ce32e83d7628 2024-11-14T06:18:29,587 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/44364c4c9f924e0bb5a0e922d2b7b373 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/44364c4c9f924e0bb5a0e922d2b7b373 2024-11-14T06:18:29,588 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/d66c8fb12b554bd6816eb537e87af4ce to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/d66c8fb12b554bd6816eb537e87af4ce 2024-11-14T06:18:29,589 DEBUG [StoreCloser-TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/e4ad394b98c74d0d8be8b9fade1fdc68 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/e4ad394b98c74d0d8be8b9fade1fdc68 2024-11-14T06:18:29,593 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/recovered.edits/498.seqid, newMaxSeqId=498, maxSeqId=1 2024-11-14T06:18:29,593 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. 2024-11-14T06:18:29,593 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1676): Region close journal for c151951bebfcfd871997dfedda0e3149: Waiting for close lock at 1731565107941Running coprocessor pre-close hooks at 1731565107941Disabling compacts and flushes for region at 1731565107941Disabling writes for close at 1731565108267 (+326 ms)Obtaining lock to block concurrent updates at 1731565108267Preparing flush snapshotting stores in c151951bebfcfd871997dfedda0e3149 at 1731565108267Finished memstore snapshotting TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1731565108267Flushing stores of TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149. at 1731565108268 (+1 ms)Flushing c151951bebfcfd871997dfedda0e3149/A: creating writer at 1731565108268Flushing c151951bebfcfd871997dfedda0e3149/A: appending metadata at 1731565108273 (+5 ms)Flushing c151951bebfcfd871997dfedda0e3149/A: closing flushed file at 1731565108273Flushing c151951bebfcfd871997dfedda0e3149/B: creating writer at 1731565108684 (+411 ms)Flushing c151951bebfcfd871997dfedda0e3149/B: appending metadata at 1731565108687 (+3 ms)Flushing c151951bebfcfd871997dfedda0e3149/B: closing flushed file at 1731565108687Flushing c151951bebfcfd871997dfedda0e3149/C: creating writer at 1731565109094 (+407 ms)Flushing c151951bebfcfd871997dfedda0e3149/C: appending metadata at 1731565109096 (+2 ms)Flushing c151951bebfcfd871997dfedda0e3149/C: closing flushed file at 1731565109096Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36c2cf56: reopening flushed file at 1731565109503 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@25fb739c: reopening flushed file at 1731565109506 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6042d8f8: reopening flushed file at 1731565109509 (+3 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for c151951bebfcfd871997dfedda0e3149 in 1246ms, sequenceid=495, compaction requested=true at 1731565109513 (+4 ms)Writing region close event to WAL at 1731565109590 (+77 ms)Running coprocessor post-close hooks at 1731565109593 (+3 ms)Closed at 1731565109593 2024-11-14T06:18:29,595 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(157): Closed c151951bebfcfd871997dfedda0e3149 2024-11-14T06:18:29,595 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=c151951bebfcfd871997dfedda0e3149, regionState=CLOSED 2024-11-14T06:18:29,597 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 because future has completed 2024-11-14T06:18:29,599 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-14T06:18:29,599 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseRegionProcedure c151951bebfcfd871997dfedda0e3149, server=ef93ef3a83c5,42283,1731564965945 in 1.8090 sec 2024-11-14T06:18:29,601 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=136 2024-11-14T06:18:29,601 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=136, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c151951bebfcfd871997dfedda0e3149, UNASSIGN in 1.8210 sec 2024-11-14T06:18:29,602 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-14T06:18:29,602 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8250 sec 2024-11-14T06:18:29,603 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731565109603"}]},"ts":"1731565109603"} 2024-11-14T06:18:29,604 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T06:18:29,604 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T06:18:29,605 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8340 sec 2024-11-14T06:18:29,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-14T06:18:29,927 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:18:29,927 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-14T06:18:29,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T06:18:29,929 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=139, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T06:18:29,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-14T06:18:29,929 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=139, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T06:18:29,931 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149 2024-11-14T06:18:29,933 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A, FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B, FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C, FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/recovered.edits] 2024-11-14T06:18:29,935 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/315e0aca3a40410db884913a02d77f50 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/315e0aca3a40410db884913a02d77f50 2024-11-14T06:18:29,937 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/b55c0444a31040a2a1396578c6e09e5d to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/b55c0444a31040a2a1396578c6e09e5d 2024-11-14T06:18:29,938 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/ca9b102709504cdca55a713315ff658c to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/A/ca9b102709504cdca55a713315ff658c 2024-11-14T06:18:29,940 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/26e5f3dc578841368e7f5715782c3f83 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/26e5f3dc578841368e7f5715782c3f83 2024-11-14T06:18:29,940 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/9378e3194df6483084ea9e9f82e009ff to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/9378e3194df6483084ea9e9f82e009ff 2024-11-14T06:18:29,941 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/c334f7826890482facdfdbacfa809d79 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/B/c334f7826890482facdfdbacfa809d79 2024-11-14T06:18:29,943 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/12245e7a8cd2483c822ff1825e97f92c to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/12245e7a8cd2483c822ff1825e97f92c 2024-11-14T06:18:29,943 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/a6e97d9b570840e38cc932ae8807b90e to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/a6e97d9b570840e38cc932ae8807b90e 2024-11-14T06:18:29,944 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/d09d82a546e24bebb892ebb72b09e1b9 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/C/d09d82a546e24bebb892ebb72b09e1b9 2024-11-14T06:18:29,946 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/recovered.edits/498.seqid to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149/recovered.edits/498.seqid 2024-11-14T06:18:29,946 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/c151951bebfcfd871997dfedda0e3149 2024-11-14T06:18:29,946 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T06:18:29,948 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=139, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T06:18:29,950 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T06:18:29,951 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T06:18:29,952 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=139, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T06:18:29,952 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T06:18:29,952 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731565109952"}]},"ts":"9223372036854775807"} 2024-11-14T06:18:29,954 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T06:18:29,954 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c151951bebfcfd871997dfedda0e3149, NAME => 'TestAcidGuarantees,,1731565084819.c151951bebfcfd871997dfedda0e3149.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T06:18:29,954 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T06:18:29,954 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731565109954"}]},"ts":"9223372036854775807"} 2024-11-14T06:18:29,955 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T06:18:29,956 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=139, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T06:18:29,957 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 29 msec 2024-11-14T06:18:30,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-14T06:18:30,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T06:18:30,038 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:18:30,046 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=250 (was 249) - Thread LEAK? -, OpenFileDescriptor=449 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=696 (was 731), ProcessCount=11 (was 11), AvailableMemoryMB=312 (was 815) 2024-11-14T06:18:30,053 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=250, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=696, ProcessCount=11, AvailableMemoryMB=312 2024-11-14T06:18:30,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T06:18:30,054 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T06:18:30,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=140, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T06:18:30,056 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T06:18:30,056 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:18:30,056 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 140 2024-11-14T06:18:30,057 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T06:18:30,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-14T06:18:30,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742374_1550 (size=1061) 2024-11-14T06:18:30,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-14T06:18:30,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-14T06:18:30,462 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02 2024-11-14T06:18:30,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742375_1551 (size=53) 2024-11-14T06:18:30,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-14T06:18:30,868 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T06:18:30,868 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing d062eac452eae155317450043cfef448, disabling compactions & flushes 2024-11-14T06:18:30,868 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:30,868 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:30,868 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. after waiting 0 ms 2024-11-14T06:18:30,868 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:30,868 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:30,868 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for d062eac452eae155317450043cfef448: Waiting for close lock at 1731565110868Disabling compacts and flushes for region at 1731565110868Disabling writes for close at 1731565110868Writing region close event to WAL at 1731565110868Closed at 1731565110868 2024-11-14T06:18:30,869 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T06:18:30,869 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731565110869"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731565110869"}]},"ts":"1731565110869"} 2024-11-14T06:18:30,871 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T06:18:30,871 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T06:18:30,872 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731565110872"}]},"ts":"1731565110872"} 2024-11-14T06:18:30,873 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T06:18:30,874 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d062eac452eae155317450043cfef448, ASSIGN}] 2024-11-14T06:18:30,875 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d062eac452eae155317450043cfef448, ASSIGN 2024-11-14T06:18:30,875 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d062eac452eae155317450043cfef448, ASSIGN; state=OFFLINE, location=ef93ef3a83c5,42283,1731564965945; forceNewPlan=false, retain=false 2024-11-14T06:18:31,026 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=d062eac452eae155317450043cfef448, regionState=OPENING, regionLocation=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:31,027 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d062eac452eae155317450043cfef448, ASSIGN because future has completed 2024-11-14T06:18:31,027 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945}] 2024-11-14T06:18:31,181 INFO [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:31,181 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(7752): Opening region: {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} 2024-11-14T06:18:31,181 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d062eac452eae155317450043cfef448 2024-11-14T06:18:31,181 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T06:18:31,181 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(7794): checking encryption for d062eac452eae155317450043cfef448 2024-11-14T06:18:31,181 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(7797): checking classloading for d062eac452eae155317450043cfef448 2024-11-14T06:18:31,183 INFO [StoreOpener-d062eac452eae155317450043cfef448-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d062eac452eae155317450043cfef448 2024-11-14T06:18:31,184 INFO [StoreOpener-d062eac452eae155317450043cfef448-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T06:18:31,184 INFO [StoreOpener-d062eac452eae155317450043cfef448-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d062eac452eae155317450043cfef448 columnFamilyName A 2024-11-14T06:18:31,184 DEBUG [StoreOpener-d062eac452eae155317450043cfef448-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:18:31,184 INFO [StoreOpener-d062eac452eae155317450043cfef448-1 {}] regionserver.HStore(327): Store=d062eac452eae155317450043cfef448/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T06:18:31,184 INFO [StoreOpener-d062eac452eae155317450043cfef448-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d062eac452eae155317450043cfef448 2024-11-14T06:18:31,185 INFO [StoreOpener-d062eac452eae155317450043cfef448-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T06:18:31,185 INFO [StoreOpener-d062eac452eae155317450043cfef448-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d062eac452eae155317450043cfef448 columnFamilyName B 2024-11-14T06:18:31,185 DEBUG [StoreOpener-d062eac452eae155317450043cfef448-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:18:31,186 INFO [StoreOpener-d062eac452eae155317450043cfef448-1 {}] regionserver.HStore(327): Store=d062eac452eae155317450043cfef448/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T06:18:31,186 INFO [StoreOpener-d062eac452eae155317450043cfef448-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d062eac452eae155317450043cfef448 2024-11-14T06:18:31,186 INFO [StoreOpener-d062eac452eae155317450043cfef448-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T06:18:31,187 INFO [StoreOpener-d062eac452eae155317450043cfef448-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d062eac452eae155317450043cfef448 columnFamilyName C 2024-11-14T06:18:31,187 DEBUG [StoreOpener-d062eac452eae155317450043cfef448-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:18:31,187 INFO [StoreOpener-d062eac452eae155317450043cfef448-1 {}] regionserver.HStore(327): Store=d062eac452eae155317450043cfef448/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T06:18:31,187 INFO [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:31,187 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1038): replaying wal for d062eac452eae155317450043cfef448 2024-11-14T06:18:31,188 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448 2024-11-14T06:18:31,188 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448 2024-11-14T06:18:31,189 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1048): stopping wal replay for d062eac452eae155317450043cfef448 2024-11-14T06:18:31,189 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1060): Cleaning up temporary data for d062eac452eae155317450043cfef448 2024-11-14T06:18:31,189 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T06:18:31,190 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1093): writing seq id for d062eac452eae155317450043cfef448 2024-11-14T06:18:31,192 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T06:18:31,192 INFO [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1114): Opened d062eac452eae155317450043cfef448; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68007410, jitterRate=0.013389378786087036}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T06:18:31,192 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d062eac452eae155317450043cfef448 2024-11-14T06:18:31,193 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1006): Region open journal for d062eac452eae155317450043cfef448: Running coprocessor pre-open hook at 1731565111182Writing region info on filesystem at 1731565111182Initializing all the Stores at 1731565111182Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731565111182Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731565111182Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731565111182Cleaning up temporary data from old regions at 1731565111189 (+7 ms)Running coprocessor post-open hooks at 1731565111192 (+3 ms)Region opened successfully at 1731565111193 (+1 ms) 2024-11-14T06:18:31,194 INFO [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., pid=142, masterSystemTime=1731565111179 2024-11-14T06:18:31,195 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:31,195 INFO [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:31,196 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=d062eac452eae155317450043cfef448, regionState=OPEN, openSeqNum=2, regionLocation=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:31,197 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 because future has completed 2024-11-14T06:18:31,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-14T06:18:31,199 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-14T06:18:31,199 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; OpenRegionProcedure d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 in 170 msec 2024-11-14T06:18:31,200 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-14T06:18:31,200 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d062eac452eae155317450043cfef448, ASSIGN in 325 msec 2024-11-14T06:18:31,200 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T06:18:31,201 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731565111201"}]},"ts":"1731565111201"} 2024-11-14T06:18:31,202 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T06:18:31,202 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T06:18:31,203 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1480 sec 2024-11-14T06:18:32,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-14T06:18:32,208 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:18:32,209 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d7f86a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:32,209 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:18:32,209 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:18:32,210 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:18:32,210 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:18:32,210 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:18:32,210 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ae1d311, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:32,210 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:18:32,211 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:18:32,211 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:32,211 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39818, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:18:32,212 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@986474, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:32,212 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:18:32,213 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:18:32,213 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:18:32,214 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52976, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:18:32,215 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T06:18:32,215 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is ef93ef3a83c5,36045,1731564965170 2024-11-14T06:18:32,215 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7e73e787 2024-11-14T06:18:32,216 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T06:18:32,217 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39834, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T06:18:32,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T06:18:32,218 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T06:18:32,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-14T06:18:32,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742376_1552 (size=1097) 2024-11-14T06:18:32,721 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-14T06:18:32,721 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-14T06:18:32,723 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T06:18:32,724 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d062eac452eae155317450043cfef448, REOPEN/MOVE}] 2024-11-14T06:18:32,725 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d062eac452eae155317450043cfef448, REOPEN/MOVE 2024-11-14T06:18:32,726 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=d062eac452eae155317450043cfef448, regionState=CLOSING, regionLocation=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:32,727 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d062eac452eae155317450043cfef448, REOPEN/MOVE because future has completed 2024-11-14T06:18:32,727 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T06:18:32,727 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945}] 2024-11-14T06:18:32,879 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(122): Close d062eac452eae155317450043cfef448 2024-11-14T06:18:32,879 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T06:18:32,879 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1722): Closing d062eac452eae155317450043cfef448, disabling compactions & flushes 2024-11-14T06:18:32,879 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:32,879 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:32,879 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. after waiting 0 ms 2024-11-14T06:18:32,879 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:32,882 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T06:18:32,883 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:32,883 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1676): Region close journal for d062eac452eae155317450043cfef448: Waiting for close lock at 1731565112879Running coprocessor pre-close hooks at 1731565112879Disabling compacts and flushes for region at 1731565112879Disabling writes for close at 1731565112879Writing region close event to WAL at 1731565112880 (+1 ms)Running coprocessor post-close hooks at 1731565112883 (+3 ms)Closed at 1731565112883 2024-11-14T06:18:32,883 WARN [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegionServer(3299): Not adding moved region record: d062eac452eae155317450043cfef448 to self. 2024-11-14T06:18:32,885 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(157): Closed d062eac452eae155317450043cfef448 2024-11-14T06:18:32,885 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=d062eac452eae155317450043cfef448, regionState=CLOSED 2024-11-14T06:18:32,886 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 because future has completed 2024-11-14T06:18:32,888 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-14T06:18:32,888 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; CloseRegionProcedure d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 in 160 msec 2024-11-14T06:18:32,888 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d062eac452eae155317450043cfef448, REOPEN/MOVE; state=CLOSED, location=ef93ef3a83c5,42283,1731564965945; forceNewPlan=false, retain=true 2024-11-14T06:18:33,039 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=d062eac452eae155317450043cfef448, regionState=OPENING, regionLocation=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:33,040 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d062eac452eae155317450043cfef448, REOPEN/MOVE because future has completed 2024-11-14T06:18:33,040 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945}] 2024-11-14T06:18:33,194 INFO [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:33,194 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(7752): Opening region: {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} 2024-11-14T06:18:33,194 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d062eac452eae155317450043cfef448 2024-11-14T06:18:33,194 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T06:18:33,195 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(7794): checking encryption for d062eac452eae155317450043cfef448 2024-11-14T06:18:33,195 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(7797): checking classloading for d062eac452eae155317450043cfef448 2024-11-14T06:18:33,196 INFO [StoreOpener-d062eac452eae155317450043cfef448-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d062eac452eae155317450043cfef448 2024-11-14T06:18:33,196 INFO [StoreOpener-d062eac452eae155317450043cfef448-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T06:18:33,196 INFO [StoreOpener-d062eac452eae155317450043cfef448-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d062eac452eae155317450043cfef448 columnFamilyName A 2024-11-14T06:18:33,197 DEBUG [StoreOpener-d062eac452eae155317450043cfef448-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:18:33,197 INFO [StoreOpener-d062eac452eae155317450043cfef448-1 {}] regionserver.HStore(327): Store=d062eac452eae155317450043cfef448/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T06:18:33,198 INFO [StoreOpener-d062eac452eae155317450043cfef448-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d062eac452eae155317450043cfef448 2024-11-14T06:18:33,198 INFO [StoreOpener-d062eac452eae155317450043cfef448-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T06:18:33,198 INFO [StoreOpener-d062eac452eae155317450043cfef448-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d062eac452eae155317450043cfef448 columnFamilyName B 2024-11-14T06:18:33,198 DEBUG [StoreOpener-d062eac452eae155317450043cfef448-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:18:33,198 INFO [StoreOpener-d062eac452eae155317450043cfef448-1 {}] regionserver.HStore(327): Store=d062eac452eae155317450043cfef448/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T06:18:33,198 INFO [StoreOpener-d062eac452eae155317450043cfef448-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d062eac452eae155317450043cfef448 2024-11-14T06:18:33,199 INFO [StoreOpener-d062eac452eae155317450043cfef448-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T06:18:33,199 INFO [StoreOpener-d062eac452eae155317450043cfef448-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d062eac452eae155317450043cfef448 columnFamilyName C 2024-11-14T06:18:33,199 DEBUG [StoreOpener-d062eac452eae155317450043cfef448-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:18:33,199 INFO [StoreOpener-d062eac452eae155317450043cfef448-1 {}] regionserver.HStore(327): Store=d062eac452eae155317450043cfef448/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T06:18:33,199 INFO [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:33,199 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1038): replaying wal for d062eac452eae155317450043cfef448 2024-11-14T06:18:33,200 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448 2024-11-14T06:18:33,200 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448 2024-11-14T06:18:33,201 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1048): stopping wal replay for d062eac452eae155317450043cfef448 2024-11-14T06:18:33,201 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1060): Cleaning up temporary data for d062eac452eae155317450043cfef448 2024-11-14T06:18:33,201 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T06:18:33,202 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1093): writing seq id for d062eac452eae155317450043cfef448 2024-11-14T06:18:33,202 INFO [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1114): Opened d062eac452eae155317450043cfef448; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65947696, jitterRate=-0.017302751541137695}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T06:18:33,203 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d062eac452eae155317450043cfef448 2024-11-14T06:18:33,203 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1006): Region open journal for d062eac452eae155317450043cfef448: Running coprocessor pre-open hook at 1731565113195Writing region info on filesystem at 1731565113195Initializing all the Stores at 1731565113195Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731565113195Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731565113195Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731565113195Cleaning up temporary data from old regions at 1731565113201 (+6 ms)Running coprocessor post-open hooks at 1731565113203 (+2 ms)Region opened successfully at 1731565113203 2024-11-14T06:18:33,203 INFO [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., pid=147, masterSystemTime=1731565113192 2024-11-14T06:18:33,205 DEBUG [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:33,205 INFO [RS_OPEN_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:33,205 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=d062eac452eae155317450043cfef448, regionState=OPEN, openSeqNum=5, regionLocation=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:33,206 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 because future has completed 2024-11-14T06:18:33,208 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=145 2024-11-14T06:18:33,208 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=145, state=SUCCESS, hasLock=false; OpenRegionProcedure d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 in 167 msec 2024-11-14T06:18:33,209 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-14T06:18:33,210 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d062eac452eae155317450043cfef448, REOPEN/MOVE in 484 msec 2024-11-14T06:18:33,211 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-14T06:18:33,211 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 487 msec 2024-11-14T06:18:33,213 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 994 msec 2024-11-14T06:18:33,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-14T06:18:33,214 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:18:33,214 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8e5803b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:33,214 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:18:33,214 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:18:33,215 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:18:33,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:18:33,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:18:33,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62043cd2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:33,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:18:33,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:18:33,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:33,216 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39856, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:18:33,216 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a49e488, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:33,217 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@438af03e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:33,217 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:18:33,217 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:18:33,217 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:18:33,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:18:33,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:18:33,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@562bc66a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:33,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:18:33,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:18:33,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:33,218 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39872, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:18:33,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d3df000, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:33,219 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dc814e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:33,219 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:18:33,219 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:18:33,219 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:18:33,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:18:33,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:18:33,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@100b199f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:33,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:18:33,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:18:33,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:33,220 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39892, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:18:33,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74731b61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:33,221 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3997c316, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:33,221 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:18:33,221 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:18:33,222 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:18:33,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:18:33,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:18:33,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59637c9f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:33,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:18:33,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:18:33,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:33,223 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39920, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:18:33,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@216116b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:33,224 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ac9387b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:33,224 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:18:33,224 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:18:33,225 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:18:33,225 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:18:33,225 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:18:33,225 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fce8bf3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:33,225 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:18:33,225 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:18:33,225 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:33,226 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39940, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:18:33,226 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29138ebd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:33,227 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@929d24a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:33,227 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:18:33,227 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:18:33,228 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:18:33,228 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:18:33,228 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:18:33,228 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b1a3f89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:33,228 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:18:33,228 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:18:33,228 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:33,229 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39958, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:18:33,229 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3229806a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:33,229 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6145e88b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:33,230 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:18:33,230 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:18:33,230 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:18:33,230 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:18:33,230 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:18:33,230 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53afbea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:33,230 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:18:33,230 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:18:33,231 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:33,231 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39982, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:18:33,231 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2581d0bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:33,232 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b47d74b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:33,232 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:18:33,232 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:18:33,233 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:18:33,233 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:18:33,233 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:18:33,233 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d9a40d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:33,233 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:18:33,233 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:18:33,233 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:33,234 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40004, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:18:33,234 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60f7e3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:33,235 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62f6d2dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:33,235 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:18:33,235 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:18:33,236 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:18:33,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:18:33,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:18:33,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73aad31b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:33,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:18:33,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:18:33,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:33,237 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40030, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:18:33,237 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30ec2d1f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:33,238 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7dcd5d59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:33,238 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef93ef3a83c5,36045,-1 for getting cluster id 2024-11-14T06:18:33,238 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T06:18:33,238 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5b31206-2687-4dcd-9a11-4e6c55e0cfc0' 2024-11-14T06:18:33,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T06:18:33,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5b31206-2687-4dcd-9a11-4e6c55e0cfc0" 2024-11-14T06:18:33,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b0a307e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:33,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef93ef3a83c5,36045,-1] 2024-11-14T06:18:33,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T06:18:33,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:33,239 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40040, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T06:18:33,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46aee1d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T06:18:33,240 DEBUG [Thread-2452 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:18:33,240 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T06:18:33,241 DEBUG [Thread-2467 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:18:33,241 DEBUG [Thread-2463 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:18:33,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:18:33,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:18:33,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-14T06:18:33,241 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:18:33,241 DEBUG [Thread-2458 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:18:33,242 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:18:33,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:18:33,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:18:33,242 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52978, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:18:33,242 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T06:18:33,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-14T06:18:33,243 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:18:33,243 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52996, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:18:33,243 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:18:33,244 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52992, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:18:33,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5] 2024-11-14T06:18:33,244 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53004, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:18:33,244 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T06:18:33,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5] 2024-11-14T06:18:33,245 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T06:18:33,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5] 2024-11-14T06:18:33,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5] 2024-11-14T06:18:33,246 DEBUG [Thread-2469 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:18:33,247 DEBUG [Thread-2450 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:18:33,247 DEBUG [Thread-2461 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:18:33,247 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:18:33,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:18:33,247 DEBUG [Thread-2465 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:18:33,247 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:18:33,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:18:33,248 DEBUG [Thread-2465 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:18:33,248 DEBUG [Thread-2465 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:18:33,249 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53016, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:18:33,249 DEBUG [Thread-2454 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:18:33,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:18:33,250 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53024, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:18:33,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:18:33,250 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:18:33,250 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53032, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:18:33,250 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:18:33,250 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53048, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:18:33,251 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5] 2024-11-14T06:18:33,251 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5] 2024-11-14T06:18:33,252 DEBUG [Thread-2456 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T06:18:33,253 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53060, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:18:33,253 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5] 2024-11-14T06:18:33,253 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5] 2024-11-14T06:18:33,255 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5] 2024-11-14T06:18:33,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on d062eac452eae155317450043cfef448 2024-11-14T06:18:33,256 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d062eac452eae155317450043cfef448 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T06:18:33,258 DEBUG [Thread-2456 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef93ef3a83c5,42283,1731564965945, seqNum=-1] 2024-11-14T06:18:33,258 DEBUG [Thread-2456 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T06:18:33,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=A 2024-11-14T06:18:33,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:33,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=B 2024-11-14T06:18:33,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:33,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=C 2024-11-14T06:18:33,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:33,260 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53070, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T06:18:33,263 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5] 2024-11-14T06:18:33,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:33,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565173277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:33,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:33,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565173278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:33,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:33,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565173279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:33,282 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:33,282 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:33,282 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:33,282 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731565113274, max timestamp=1731565113279Num uniques 859; 2024-11-14T06:18:33,282 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T06:18:33,282 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731565113274, max timestamp=1731565113279Num uniques 859; 2024-11-14T06:18:33,282 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T06:18:33,283 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:33,283 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:33,283 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:33,283 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:33,283 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:33,283 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:33,285 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731565113274, max timestamp=1731565113279Num uniques 859; 2024-11-14T06:18:33,285 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-14T06:18:33,297 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114d311f2a85c9d4e969b3f2ca0f0c6ef05_d062eac452eae155317450043cfef448 is 50, key is test_row_0/A:col10/1731565113251/Put/seqid=0 2024-11-14T06:18:33,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742377_1553 (size=12154) 2024-11-14T06:18:33,328 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:18:33,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:33,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565173323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:33,328 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:33,329 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:33,329 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:33,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:33,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52978 deadline: 1731565173328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:33,332 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114d311f2a85c9d4e969b3f2ca0f0c6ef05_d062eac452eae155317450043cfef448 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d311f2a85c9d4e969b3f2ca0f0c6ef05_d062eac452eae155317450043cfef448 2024-11-14T06:18:33,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:33,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:33,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:33,332 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/5ad92034a9434cd38febcd8ce885cdef, store: [table=TestAcidGuarantees family=A region=d062eac452eae155317450043cfef448] 2024-11-14T06:18:33,333 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/5ad92034a9434cd38febcd8ce885cdef is 175, key is test_row_0/A:col10/1731565113251/Put/seqid=0 2024-11-14T06:18:33,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-14T06:18:33,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742378_1554 (size=30955) 2024-11-14T06:18:33,351 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/5ad92034a9434cd38febcd8ce885cdef 2024-11-14T06:18:33,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/07a79771b9594786ade9fce2aa331319 is 50, key is test_row_0/B:col10/1731565113251/Put/seqid=0 2024-11-14T06:18:33,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:33,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565173387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:33,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:33,392 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:33,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565173388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:33,392 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:33,392 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:33,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:33,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:33,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:33,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:33,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565173388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:33,393 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:33,393 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:33,393 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:33,398 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-14T06:18:33,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:33,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. as already flushing 2024-11-14T06:18:33,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:33,398 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:33,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:33,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:33,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742379_1555 (size=12001) 2024-11-14T06:18:33,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:33,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565173437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:33,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:33,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52978 deadline: 1731565173437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:33,441 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:33,441 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:33,441 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:33,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:33,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:33,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:33,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-14T06:18:33,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:33,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. as already flushing 2024-11-14T06:18:33,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:33,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:33,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:33,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:33,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-14T06:18:33,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:33,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565173597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:33,598 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:33,598 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:33,598 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:33,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:33,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565173600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:33,604 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:33,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:33,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:33,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:33,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565173605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:33,607 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:33,607 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:33,607 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:33,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:33,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52978 deadline: 1731565173648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:33,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:33,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565173648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:33,652 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:33,652 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:33,652 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:33,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:33,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:33,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:33,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-14T06:18:33,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:33,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. as already flushing 2024-11-14T06:18:33,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:33,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:33,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:33,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:33,812 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/07a79771b9594786ade9fce2aa331319 2024-11-14T06:18:33,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/97f94f010c7049d692542d3d9261971a is 50, key is test_row_0/C:col10/1731565113251/Put/seqid=0 2024-11-14T06:18:33,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742380_1556 (size=12001) 2024-11-14T06:18:33,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-14T06:18:33,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:33,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. as already flushing 2024-11-14T06:18:33,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:33,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:33,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:33,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:33,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-14T06:18:33,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:33,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565173907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:33,908 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:33,908 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:33,908 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:33,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:33,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565173917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:33,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:33,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:33,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:33,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:33,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565173917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:33,920 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:33,920 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:33,920 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:33,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:33,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:33,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565173957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:33,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52978 deadline: 1731565173957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:33,964 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:33,964 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:33,964 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:33,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:33,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:33,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:34,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-14T06:18:34,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:34,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. as already flushing 2024-11-14T06:18:34,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:34,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:34,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:34,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:34,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-14T06:18:34,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:34,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. as already flushing 2024-11-14T06:18:34,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:34,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:34,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:34,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:34,242 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/97f94f010c7049d692542d3d9261971a 2024-11-14T06:18:34,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/5ad92034a9434cd38febcd8ce885cdef as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/5ad92034a9434cd38febcd8ce885cdef 2024-11-14T06:18:34,258 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/5ad92034a9434cd38febcd8ce885cdef, entries=150, sequenceid=15, filesize=30.2 K 2024-11-14T06:18:34,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/07a79771b9594786ade9fce2aa331319 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/07a79771b9594786ade9fce2aa331319 2024-11-14T06:18:34,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/07a79771b9594786ade9fce2aa331319, entries=150, sequenceid=15, filesize=11.7 K 2024-11-14T06:18:34,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/97f94f010c7049d692542d3d9261971a as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/97f94f010c7049d692542d3d9261971a 2024-11-14T06:18:34,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/97f94f010c7049d692542d3d9261971a, entries=150, sequenceid=15, filesize=11.7 K 2024-11-14T06:18:34,267 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=129.31 KB/132415 for d062eac452eae155317450043cfef448 in 1011ms, sequenceid=15, compaction requested=false 2024-11-14T06:18:34,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d062eac452eae155317450043cfef448: 2024-11-14T06:18:34,283 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T06:18:34,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-14T06:18:34,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:34,316 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing d062eac452eae155317450043cfef448 3/3 column families, dataSize=129.31 KB heapSize=364.54 KB 2024-11-14T06:18:34,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=A 2024-11-14T06:18:34,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:18:34,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=B 2024-11-14T06:18:34,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:18:34,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=C 2024-11-14T06:18:34,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:18:34,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114f4914fafc4ab4f739d6f5a74e77d8b78_d062eac452eae155317450043cfef448 is 50, key is test_row_0/A:col10/1731565113321/Put/seqid=0 2024-11-14T06:18:34,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742381_1557 (size=12154) 2024-11-14T06:18:34,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:18:34,328 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114f4914fafc4ab4f739d6f5a74e77d8b78_d062eac452eae155317450043cfef448 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f4914fafc4ab4f739d6f5a74e77d8b78_d062eac452eae155317450043cfef448 2024-11-14T06:18:34,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/d2d29244d19f4389aaf0a062f51f99e4, store: [table=TestAcidGuarantees family=A region=d062eac452eae155317450043cfef448] 2024-11-14T06:18:34,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/d2d29244d19f4389aaf0a062f51f99e4 is 175, key is test_row_0/A:col10/1731565113321/Put/seqid=0 2024-11-14T06:18:34,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742382_1558 (size=30955) 2024-11-14T06:18:34,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-14T06:18:34,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on d062eac452eae155317450043cfef448 2024-11-14T06:18:34,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. as already flushing 2024-11-14T06:18:34,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:34,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565174437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:34,443 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:34,443 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:34,443 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:34,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:34,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565174438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:34,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:34,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:34,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:34,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:34,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565174439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:34,444 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:34,444 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:34,444 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:34,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:34,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52978 deadline: 1731565174467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:34,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:34,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:34,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:34,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:34,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565174477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:34,480 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:34,480 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:34,480 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:34,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:34,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565174547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:34,551 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:34,551 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:34,551 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:34,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:34,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565174548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:34,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:34,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:34,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:34,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:34,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565174548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:34,551 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:34,552 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:34,552 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:34,735 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/d2d29244d19f4389aaf0a062f51f99e4 2024-11-14T06:18:34,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/027ad5a6fc3249999a0f861f4fa1c034 is 50, key is test_row_0/B:col10/1731565113321/Put/seqid=0 2024-11-14T06:18:34,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:34,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565174770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:34,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:34,775 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:34,775 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:34,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565174770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:34,775 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:34,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:34,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:34,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:34,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:34,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565174770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:34,776 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:34,776 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:34,776 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:34,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742383_1559 (size=12001) 2024-11-14T06:18:34,792 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/027ad5a6fc3249999a0f861f4fa1c034 2024-11-14T06:18:34,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/d6340760470b4080883adfedf44732b5 is 50, key is test_row_0/C:col10/1731565113321/Put/seqid=0 2024-11-14T06:18:34,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742384_1560 (size=12001) 2024-11-14T06:18:35,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:35,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565175088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:35,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:35,094 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:35,094 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:35,094 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:35,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565175088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:35,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:35,094 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:35,094 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:35,094 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:35,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565175088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:35,094 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:35,094 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:35,094 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:35,123 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T06:18:35,247 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=39.86 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/d6340760470b4080883adfedf44732b5 2024-11-14T06:18:35,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/d2d29244d19f4389aaf0a062f51f99e4 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/d2d29244d19f4389aaf0a062f51f99e4 2024-11-14T06:18:35,274 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/d2d29244d19f4389aaf0a062f51f99e4, entries=150, sequenceid=50, filesize=30.2 K 2024-11-14T06:18:35,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/027ad5a6fc3249999a0f861f4fa1c034 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/027ad5a6fc3249999a0f861f4fa1c034 2024-11-14T06:18:35,287 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/027ad5a6fc3249999a0f861f4fa1c034, entries=150, sequenceid=50, filesize=11.7 K 2024-11-14T06:18:35,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/d6340760470b4080883adfedf44732b5 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/d6340760470b4080883adfedf44732b5 2024-11-14T06:18:35,294 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/d6340760470b4080883adfedf44732b5, entries=150, sequenceid=50, filesize=11.7 K 2024-11-14T06:18:35,295 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~129.31 KB/132415, heapSize ~364.49 KB/373240, currentSize=60.38 KB/61830 for d062eac452eae155317450043cfef448 in 979ms, sequenceid=50, compaction requested=false 2024-11-14T06:18:35,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for d062eac452eae155317450043cfef448: 2024-11-14T06:18:35,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:35,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-14T06:18:35,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-14T06:18:35,304 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-14T06:18:35,304 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0560 sec 2024-11-14T06:18:35,308 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 2.0660 sec 2024-11-14T06:18:35,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-14T06:18:35,387 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:18:35,388 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T06:18:35,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-14T06:18:35,390 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T06:18:35,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-14T06:18:35,391 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T06:18:35,391 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T06:18:35,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on d062eac452eae155317450043cfef448 2024-11-14T06:18:35,491 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d062eac452eae155317450043cfef448 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-14T06:18:35,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=A 2024-11-14T06:18:35,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:35,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=B 2024-11-14T06:18:35,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:35,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=C 2024-11-14T06:18:35,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:35,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-14T06:18:35,504 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111401b60a1cc7f34bb28043d9171ca6fbfe_d062eac452eae155317450043cfef448 is 50, key is test_row_0/A:col10/1731565114430/Put/seqid=0 2024-11-14T06:18:35,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742385_1561 (size=12154) 2024-11-14T06:18:35,518 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:18:35,521 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111401b60a1cc7f34bb28043d9171ca6fbfe_d062eac452eae155317450043cfef448 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111401b60a1cc7f34bb28043d9171ca6fbfe_d062eac452eae155317450043cfef448 2024-11-14T06:18:35,522 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/c9c34a8f8f3e4a6192a6a07173e2df10, store: [table=TestAcidGuarantees family=A region=d062eac452eae155317450043cfef448] 2024-11-14T06:18:35,522 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/c9c34a8f8f3e4a6192a6a07173e2df10 is 175, key is test_row_0/A:col10/1731565114430/Put/seqid=0 2024-11-14T06:18:35,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742386_1562 (size=30955) 2024-11-14T06:18:35,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-14T06:18:35,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:35,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. as already flushing 2024-11-14T06:18:35,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:35,545 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:35,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:35,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:35,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:35,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:35,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565175564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:35,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52978 deadline: 1731565175564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:35,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:35,570 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:35,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:35,570 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:35,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:35,570 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:35,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:35,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565175598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:35,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:35,601 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:35,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565175598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:35,601 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:35,601 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:35,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:35,602 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:35,602 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:35,602 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:35,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565175598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:35,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:35,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:35,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:35,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:35,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52978 deadline: 1731565175677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:35,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:35,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:35,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:35,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565175677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:35,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:35,679 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:35,679 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:35,679 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:35,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-14T06:18:35,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:35,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. as already flushing 2024-11-14T06:18:35,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:35,698 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:35,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:35,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:35,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-14T06:18:35,850 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-14T06:18:35,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:35,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. as already flushing 2024-11-14T06:18:35,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:35,851 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:35,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:35,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:35,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:35,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52978 deadline: 1731565175887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:35,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:35,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:35,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:35,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:35,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565175887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:35,891 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:35,891 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:35,891 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:35,934 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=63, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/c9c34a8f8f3e4a6192a6a07173e2df10 2024-11-14T06:18:35,939 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/437b5a7ac29d426aab7cfcc2d30329c4 is 50, key is test_row_0/B:col10/1731565114430/Put/seqid=0 2024-11-14T06:18:35,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742387_1563 (size=12001) 2024-11-14T06:18:36,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-14T06:18:36,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:36,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. as already flushing 2024-11-14T06:18:36,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:36,003 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:36,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:36,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:36,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-14T06:18:36,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-14T06:18:36,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:36,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. as already flushing 2024-11-14T06:18:36,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:36,156 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:36,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:36,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:36,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:36,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52978 deadline: 1731565176198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:36,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:36,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:36,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:36,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565176200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:36,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:36,202 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:36,202 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:36,202 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:36,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-14T06:18:36,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:36,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. as already flushing 2024-11-14T06:18:36,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:36,309 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:36,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:36,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:36,344 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/437b5a7ac29d426aab7cfcc2d30329c4 2024-11-14T06:18:36,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/2d57cc5dc0264ec19458ce5a53e5c191 is 50, key is test_row_0/C:col10/1731565114430/Put/seqid=0 2024-11-14T06:18:36,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742388_1564 (size=12001) 2024-11-14T06:18:36,368 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/2d57cc5dc0264ec19458ce5a53e5c191 2024-11-14T06:18:36,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/c9c34a8f8f3e4a6192a6a07173e2df10 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/c9c34a8f8f3e4a6192a6a07173e2df10 2024-11-14T06:18:36,375 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/c9c34a8f8f3e4a6192a6a07173e2df10, entries=150, sequenceid=63, filesize=30.2 K 2024-11-14T06:18:36,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/437b5a7ac29d426aab7cfcc2d30329c4 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/437b5a7ac29d426aab7cfcc2d30329c4 2024-11-14T06:18:36,380 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/437b5a7ac29d426aab7cfcc2d30329c4, entries=150, sequenceid=63, filesize=11.7 K 2024-11-14T06:18:36,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/2d57cc5dc0264ec19458ce5a53e5c191 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/2d57cc5dc0264ec19458ce5a53e5c191 2024-11-14T06:18:36,387 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/2d57cc5dc0264ec19458ce5a53e5c191, entries=150, sequenceid=63, filesize=11.7 K 2024-11-14T06:18:36,388 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for d062eac452eae155317450043cfef448 in 898ms, sequenceid=63, compaction requested=true 2024-11-14T06:18:36,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d062eac452eae155317450043cfef448: 2024-11-14T06:18:36,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d062eac452eae155317450043cfef448:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T06:18:36,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:36,389 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:18:36,389 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:18:36,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d062eac452eae155317450043cfef448:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T06:18:36,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:36,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d062eac452eae155317450043cfef448:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T06:18:36,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:18:36,392 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:18:36,392 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): d062eac452eae155317450043cfef448/A is initiating minor compaction (all files) 2024-11-14T06:18:36,392 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d062eac452eae155317450043cfef448/A in TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:36,392 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/5ad92034a9434cd38febcd8ce885cdef, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/d2d29244d19f4389aaf0a062f51f99e4, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/c9c34a8f8f3e4a6192a6a07173e2df10] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp, totalSize=90.7 K 2024-11-14T06:18:36,393 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:36,393 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. files: [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/5ad92034a9434cd38febcd8ce885cdef, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/d2d29244d19f4389aaf0a062f51f99e4, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/c9c34a8f8f3e4a6192a6a07173e2df10] 2024-11-14T06:18:36,393 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:18:36,393 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): d062eac452eae155317450043cfef448/B is initiating minor compaction (all files) 2024-11-14T06:18:36,393 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d062eac452eae155317450043cfef448/B in TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:36,393 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/07a79771b9594786ade9fce2aa331319, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/027ad5a6fc3249999a0f861f4fa1c034, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/437b5a7ac29d426aab7cfcc2d30329c4] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp, totalSize=35.2 K 2024-11-14T06:18:36,400 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 07a79771b9594786ade9fce2aa331319, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731565113251 2024-11-14T06:18:36,400 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ad92034a9434cd38febcd8ce885cdef, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731565113251 2024-11-14T06:18:36,401 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2d29244d19f4389aaf0a062f51f99e4, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731565113318 2024-11-14T06:18:36,401 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 027ad5a6fc3249999a0f861f4fa1c034, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731565113318 2024-11-14T06:18:36,401 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9c34a8f8f3e4a6192a6a07173e2df10, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731565114418 2024-11-14T06:18:36,402 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 437b5a7ac29d426aab7cfcc2d30329c4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731565114418 2024-11-14T06:18:36,413 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d062eac452eae155317450043cfef448] 2024-11-14T06:18:36,422 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d062eac452eae155317450043cfef448#B#compaction#484 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:18:36,423 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/a905ce88736c4c518b72d5adf4544e8f is 50, key is test_row_0/B:col10/1731565114430/Put/seqid=0 2024-11-14T06:18:36,428 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411142815b8d2fe5b482eb96081bc27be4590_d062eac452eae155317450043cfef448 store=[table=TestAcidGuarantees family=A region=d062eac452eae155317450043cfef448] 2024-11-14T06:18:36,430 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411142815b8d2fe5b482eb96081bc27be4590_d062eac452eae155317450043cfef448, store=[table=TestAcidGuarantees family=A region=d062eac452eae155317450043cfef448] 2024-11-14T06:18:36,430 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411142815b8d2fe5b482eb96081bc27be4590_d062eac452eae155317450043cfef448 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d062eac452eae155317450043cfef448] 2024-11-14T06:18:36,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742389_1565 (size=12104) 2024-11-14T06:18:36,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742390_1566 (size=4469) 2024-11-14T06:18:36,447 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d062eac452eae155317450043cfef448#A#compaction#483 average throughput is 0.72 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:18:36,447 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/673cd1e90d224b348e6b7780ced448fa is 175, key is test_row_0/A:col10/1731565114430/Put/seqid=0 2024-11-14T06:18:36,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742391_1567 (size=31058) 2024-11-14T06:18:36,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-14T06:18:36,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:36,463 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing d062eac452eae155317450043cfef448 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-14T06:18:36,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=A 2024-11-14T06:18:36,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:36,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=B 2024-11-14T06:18:36,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:36,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=C 2024-11-14T06:18:36,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:36,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411140efc4da5949f4e61a34936943f1caa28_d062eac452eae155317450043cfef448 is 50, key is test_row_0/A:col10/1731565115555/Put/seqid=0 2024-11-14T06:18:36,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742392_1568 (size=12154) 2024-11-14T06:18:36,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-14T06:18:36,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on d062eac452eae155317450043cfef448 2024-11-14T06:18:36,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. as already flushing 2024-11-14T06:18:36,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:36,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565176623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:36,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:36,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:36,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:36,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:36,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565176627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:36,632 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:36,632 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:36,632 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:36,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:36,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565176628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:36,635 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:36,635 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:36,635 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:36,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:36,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52978 deadline: 1731565176717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:36,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:36,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:36,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:36,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:36,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565176717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:36,721 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:36,721 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:36,721 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:36,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:36,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565176737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:36,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:36,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:36,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:36,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:36,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565176737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:36,742 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:36,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:36,742 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:36,742 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:36,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565176737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:36,742 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:36,742 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:36,742 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:36,837 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/a905ce88736c4c518b72d5adf4544e8f as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/a905ce88736c4c518b72d5adf4544e8f 2024-11-14T06:18:36,840 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d062eac452eae155317450043cfef448/B of d062eac452eae155317450043cfef448 into a905ce88736c4c518b72d5adf4544e8f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:18:36,840 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d062eac452eae155317450043cfef448: 2024-11-14T06:18:36,840 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., storeName=d062eac452eae155317450043cfef448/B, priority=13, startTime=1731565116389; duration=0sec 2024-11-14T06:18:36,840 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:18:36,840 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d062eac452eae155317450043cfef448:B 2024-11-14T06:18:36,840 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:18:36,840 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:18:36,840 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): d062eac452eae155317450043cfef448/C is initiating minor compaction (all files) 2024-11-14T06:18:36,841 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d062eac452eae155317450043cfef448/C in TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:36,841 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/97f94f010c7049d692542d3d9261971a, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/d6340760470b4080883adfedf44732b5, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/2d57cc5dc0264ec19458ce5a53e5c191] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp, totalSize=35.2 K 2024-11-14T06:18:36,841 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 97f94f010c7049d692542d3d9261971a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731565113251 2024-11-14T06:18:36,841 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting d6340760470b4080883adfedf44732b5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731565113318 2024-11-14T06:18:36,841 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d57cc5dc0264ec19458ce5a53e5c191, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731565114418 2024-11-14T06:18:36,847 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d062eac452eae155317450043cfef448#C#compaction#486 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:18:36,847 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/fbbae768cd9a49f3a6d2a88189c1c206 is 50, key is test_row_0/C:col10/1731565114430/Put/seqid=0 2024-11-14T06:18:36,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742393_1569 (size=12104) 2024-11-14T06:18:36,853 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/fbbae768cd9a49f3a6d2a88189c1c206 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/fbbae768cd9a49f3a6d2a88189c1c206 2024-11-14T06:18:36,857 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d062eac452eae155317450043cfef448/C of d062eac452eae155317450043cfef448 into fbbae768cd9a49f3a6d2a88189c1c206(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:18:36,857 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d062eac452eae155317450043cfef448: 2024-11-14T06:18:36,857 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., storeName=d062eac452eae155317450043cfef448/C, priority=13, startTime=1731565116389; duration=0sec 2024-11-14T06:18:36,857 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:36,857 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d062eac452eae155317450043cfef448:C 2024-11-14T06:18:36,858 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/673cd1e90d224b348e6b7780ced448fa as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/673cd1e90d224b348e6b7780ced448fa 2024-11-14T06:18:36,861 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d062eac452eae155317450043cfef448/A of d062eac452eae155317450043cfef448 into 673cd1e90d224b348e6b7780ced448fa(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:18:36,861 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d062eac452eae155317450043cfef448: 2024-11-14T06:18:36,861 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., storeName=d062eac452eae155317450043cfef448/A, priority=13, startTime=1731565116388; duration=0sec 2024-11-14T06:18:36,861 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:36,861 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d062eac452eae155317450043cfef448:A 2024-11-14T06:18:36,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:18:36,913 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411140efc4da5949f4e61a34936943f1caa28_d062eac452eae155317450043cfef448 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411140efc4da5949f4e61a34936943f1caa28_d062eac452eae155317450043cfef448 2024-11-14T06:18:36,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/4d778e54da8c4967bc3d2ee1546d2145, store: [table=TestAcidGuarantees family=A region=d062eac452eae155317450043cfef448] 2024-11-14T06:18:36,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/4d778e54da8c4967bc3d2ee1546d2145 is 175, key is test_row_0/A:col10/1731565115555/Put/seqid=0 2024-11-14T06:18:36,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742394_1570 (size=30955) 2024-11-14T06:18:36,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:36,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565176947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:36,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:36,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:36,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:36,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:36,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565176947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:36,951 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:36,951 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:36,951 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:36,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:36,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565176947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:36,951 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:36,951 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:36,951 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:37,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:37,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565177257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:37,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:37,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565177257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:37,261 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:37,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:37,261 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:37,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:37,261 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:37,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:37,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:37,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565177257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:37,261 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:37,261 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:37,261 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:37,320 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=83, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/4d778e54da8c4967bc3d2ee1546d2145 2024-11-14T06:18:37,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/7d986465ea094ff19142a57d7b6b8198 is 50, key is test_row_0/B:col10/1731565115555/Put/seqid=0 2024-11-14T06:18:37,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742395_1571 (size=12001) 2024-11-14T06:18:37,338 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/7d986465ea094ff19142a57d7b6b8198 2024-11-14T06:18:37,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/6c5fd27fa1e3463e8bc8e4d96dd09edb is 50, key is test_row_0/C:col10/1731565115555/Put/seqid=0 2024-11-14T06:18:37,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742396_1572 (size=12001) 2024-11-14T06:18:37,354 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/6c5fd27fa1e3463e8bc8e4d96dd09edb 2024-11-14T06:18:37,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/4d778e54da8c4967bc3d2ee1546d2145 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/4d778e54da8c4967bc3d2ee1546d2145 2024-11-14T06:18:37,362 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/4d778e54da8c4967bc3d2ee1546d2145, entries=150, sequenceid=83, filesize=30.2 K 2024-11-14T06:18:37,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/7d986465ea094ff19142a57d7b6b8198 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/7d986465ea094ff19142a57d7b6b8198 2024-11-14T06:18:37,367 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/7d986465ea094ff19142a57d7b6b8198, entries=150, sequenceid=83, filesize=11.7 K 2024-11-14T06:18:37,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/6c5fd27fa1e3463e8bc8e4d96dd09edb as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/6c5fd27fa1e3463e8bc8e4d96dd09edb 2024-11-14T06:18:37,371 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/6c5fd27fa1e3463e8bc8e4d96dd09edb, entries=150, sequenceid=83, filesize=11.7 K 2024-11-14T06:18:37,372 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for d062eac452eae155317450043cfef448 in 909ms, sequenceid=83, compaction requested=false 2024-11-14T06:18:37,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for d062eac452eae155317450043cfef448: 2024-11-14T06:18:37,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:37,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-14T06:18:37,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-14T06:18:37,375 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-14T06:18:37,375 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9820 sec 2024-11-14T06:18:37,378 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 1.9880 sec 2024-11-14T06:18:37,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-14T06:18:37,538 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:18:37,538 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T06:18:37,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-14T06:18:37,539 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T06:18:37,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-14T06:18:37,540 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T06:18:37,540 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T06:18:37,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-14T06:18:37,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-14T06:18:37,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:37,692 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing d062eac452eae155317450043cfef448 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-14T06:18:37,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=A 2024-11-14T06:18:37,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:37,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=B 2024-11-14T06:18:37,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:37,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=C 2024-11-14T06:18:37,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:37,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114d4e4e01a8fd744e1af77c008aeb33b01_d062eac452eae155317450043cfef448 is 50, key is test_row_0/A:col10/1731565116626/Put/seqid=0 2024-11-14T06:18:37,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on d062eac452eae155317450043cfef448 2024-11-14T06:18:37,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. as already flushing 2024-11-14T06:18:37,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742397_1573 (size=12154) 2024-11-14T06:18:37,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:37,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565177781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:37,787 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:37,787 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:37,788 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:37,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:37,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:37,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52978 deadline: 1731565177786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:37,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565177786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:37,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:37,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:37,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:37,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:37,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565177787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:37,792 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:37,792 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:37,792 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:37,792 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:37,792 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:37,792 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:37,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:37,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565177787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:37,792 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:37,792 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:37,792 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:37,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-14T06:18:37,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:37,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565177897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:37,900 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:37,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:37,900 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:37,900 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:37,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565177897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:37,900 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:37,900 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:37,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:37,900 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:37,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565177897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:37,900 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:37,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:37,901 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:37,901 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:37,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565177897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:37,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:37,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:37,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:37,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:37,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52978 deadline: 1731565177898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:37,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:37,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:37,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:38,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:38,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565178107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:38,110 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:38,110 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:38,110 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:38,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:38,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565178107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:38,111 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:38,111 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:38,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:38,111 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:38,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565178107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:38,111 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:38,111 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:38,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:38,111 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:38,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52978 deadline: 1731565178108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:38,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:38,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:38,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:38,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:38,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565178108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:38,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:38,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:38,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:38,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:18:38,143 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114d4e4e01a8fd744e1af77c008aeb33b01_d062eac452eae155317450043cfef448 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d4e4e01a8fd744e1af77c008aeb33b01_d062eac452eae155317450043cfef448 2024-11-14T06:18:38,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/eeed945c9aa543a585c59b924ee68eaa, store: [table=TestAcidGuarantees family=A region=d062eac452eae155317450043cfef448] 2024-11-14T06:18:38,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/eeed945c9aa543a585c59b924ee68eaa is 175, key is test_row_0/A:col10/1731565116626/Put/seqid=0 2024-11-14T06:18:38,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-14T06:18:38,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742398_1574 (size=30955) 2024-11-14T06:18:38,194 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=98, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/eeed945c9aa543a585c59b924ee68eaa 2024-11-14T06:18:38,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/938a72eaff8848d680dd98c120f2c359 is 50, key is test_row_0/B:col10/1731565116626/Put/seqid=0 2024-11-14T06:18:38,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742399_1575 (size=12001) 2024-11-14T06:18:38,235 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/938a72eaff8848d680dd98c120f2c359 2024-11-14T06:18:38,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/9c81ad76ee1c4f15a191540de38bea88 is 50, key is test_row_0/C:col10/1731565116626/Put/seqid=0 2024-11-14T06:18:38,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742400_1576 (size=12001) 2024-11-14T06:18:38,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:38,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:38,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52978 deadline: 1731565178417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:38,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565178417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:38,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:38,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:38,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:38,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:38,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565178417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:38,422 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:38,422 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:38,422 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:38,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:38,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:38,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:38,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:38,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565178417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:38,422 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:38,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:38,422 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:38,422 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:38,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565178418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:38,422 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:38,422 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:38,423 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:38,660 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/9c81ad76ee1c4f15a191540de38bea88 2024-11-14T06:18:38,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/eeed945c9aa543a585c59b924ee68eaa as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/eeed945c9aa543a585c59b924ee68eaa 2024-11-14T06:18:38,677 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/eeed945c9aa543a585c59b924ee68eaa, entries=150, sequenceid=98, filesize=30.2 K 2024-11-14T06:18:38,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-14T06:18:38,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/938a72eaff8848d680dd98c120f2c359 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/938a72eaff8848d680dd98c120f2c359 2024-11-14T06:18:38,688 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/938a72eaff8848d680dd98c120f2c359, entries=150, sequenceid=98, filesize=11.7 K 2024-11-14T06:18:38,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/9c81ad76ee1c4f15a191540de38bea88 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/9c81ad76ee1c4f15a191540de38bea88 2024-11-14T06:18:38,692 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/9c81ad76ee1c4f15a191540de38bea88, entries=150, sequenceid=98, filesize=11.7 K 2024-11-14T06:18:38,693 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for d062eac452eae155317450043cfef448 in 1001ms, sequenceid=98, compaction requested=true 2024-11-14T06:18:38,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for d062eac452eae155317450043cfef448: 2024-11-14T06:18:38,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:38,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-14T06:18:38,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-14T06:18:38,701 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-14T06:18:38,701 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1540 sec 2024-11-14T06:18:38,703 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.1630 sec 2024-11-14T06:18:38,931 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d062eac452eae155317450043cfef448 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-14T06:18:38,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=A 2024-11-14T06:18:38,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:18:38,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=B 2024-11-14T06:18:38,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:18:38,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=C 2024-11-14T06:18:38,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:18:38,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on d062eac452eae155317450043cfef448 2024-11-14T06:18:38,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:38,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565178942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:38,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:38,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565178942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:38,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:38,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52978 deadline: 1731565178943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:38,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:38,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565178943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:38,950 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:38,950 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:38,950 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:38,949 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:38,950 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:38,950 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:38,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411146d860e78b01e4679878212778c20d652_d062eac452eae155317450043cfef448 is 50, key is test_row_0/A:col10/1731565117779/Put/seqid=0 2024-11-14T06:18:38,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:38,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:38,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:38,950 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:38,950 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:38,951 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:38,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:38,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565178948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:38,955 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:38,955 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:38,955 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:38,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742401_1577 (size=12154) 2024-11-14T06:18:39,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:39,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565179057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:39,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:39,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565179057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:39,062 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:39,062 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:39,062 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:39,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:39,062 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:39,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565179058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:39,062 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:39,062 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:39,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:39,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:39,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:39,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:39,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565179058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:39,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:39,063 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:39,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52978 deadline: 1731565179058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:39,063 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:39,063 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:39,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:39,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:39,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:39,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:39,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565179267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:39,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:39,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:39,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:39,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:39,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52978 deadline: 1731565179267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:39,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:39,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:39,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:39,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565179267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:39,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:39,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:39,272 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:39,272 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:39,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565179267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:39,272 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:39,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:39,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565179267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:39,273 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:39,273 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:39,273 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:39,275 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:39,275 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:39,275 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:39,363 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:18:39,366 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411146d860e78b01e4679878212778c20d652_d062eac452eae155317450043cfef448 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411146d860e78b01e4679878212778c20d652_d062eac452eae155317450043cfef448 2024-11-14T06:18:39,366 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/193507e26d9d44ec9e0fde2b26b73e92, store: [table=TestAcidGuarantees family=A region=d062eac452eae155317450043cfef448] 2024-11-14T06:18:39,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/193507e26d9d44ec9e0fde2b26b73e92 is 175, key is test_row_0/A:col10/1731565117779/Put/seqid=0 2024-11-14T06:18:39,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742402_1578 (size=30955) 2024-11-14T06:18:39,370 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=119, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/193507e26d9d44ec9e0fde2b26b73e92 2024-11-14T06:18:39,376 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/7d8bc2b45817426f9a8d4bff092c1e6b is 50, key is test_row_0/B:col10/1731565117779/Put/seqid=0 2024-11-14T06:18:39,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742403_1579 (size=12001) 2024-11-14T06:18:39,389 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/7d8bc2b45817426f9a8d4bff092c1e6b 2024-11-14T06:18:39,396 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/0c2dc43b929a4bf088df1ca94482af1d is 50, key is test_row_0/C:col10/1731565117779/Put/seqid=0 2024-11-14T06:18:39,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742404_1580 (size=12001) 2024-11-14T06:18:39,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:39,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565179577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:39,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:39,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:39,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52978 deadline: 1731565179577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:39,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:39,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:39,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:39,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:39,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:39,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:39,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565179577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:39,580 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:39,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:39,580 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:39,580 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:39,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565179578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:39,580 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:39,580 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:39,580 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:39,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:39,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565179587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:39,590 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:39,590 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:39,590 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:39,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-14T06:18:39,688 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:18:39,688 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T06:18:39,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-14T06:18:39,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-14T06:18:39,691 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T06:18:39,692 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T06:18:39,692 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T06:18:39,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-14T06:18:39,799 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/0c2dc43b929a4bf088df1ca94482af1d 2024-11-14T06:18:39,802 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/193507e26d9d44ec9e0fde2b26b73e92 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/193507e26d9d44ec9e0fde2b26b73e92 2024-11-14T06:18:39,805 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/193507e26d9d44ec9e0fde2b26b73e92, entries=150, sequenceid=119, filesize=30.2 K 2024-11-14T06:18:39,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/7d8bc2b45817426f9a8d4bff092c1e6b as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/7d8bc2b45817426f9a8d4bff092c1e6b 2024-11-14T06:18:39,809 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/7d8bc2b45817426f9a8d4bff092c1e6b, entries=150, sequenceid=119, filesize=11.7 K 2024-11-14T06:18:39,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/0c2dc43b929a4bf088df1ca94482af1d as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/0c2dc43b929a4bf088df1ca94482af1d 2024-11-14T06:18:39,813 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/0c2dc43b929a4bf088df1ca94482af1d, entries=150, sequenceid=119, filesize=11.7 K 2024-11-14T06:18:39,814 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for d062eac452eae155317450043cfef448 in 883ms, sequenceid=119, compaction requested=true 2024-11-14T06:18:39,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d062eac452eae155317450043cfef448: 2024-11-14T06:18:39,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d062eac452eae155317450043cfef448:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T06:18:39,814 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T06:18:39,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:39,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d062eac452eae155317450043cfef448:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T06:18:39,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:39,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d062eac452eae155317450043cfef448:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T06:18:39,814 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T06:18:39,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:18:39,815 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T06:18:39,815 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): d062eac452eae155317450043cfef448/B is initiating minor compaction (all files) 2024-11-14T06:18:39,815 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 123923 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T06:18:39,815 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d062eac452eae155317450043cfef448/B in TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:39,815 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): d062eac452eae155317450043cfef448/A is initiating minor compaction (all files) 2024-11-14T06:18:39,815 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d062eac452eae155317450043cfef448/A in TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:39,815 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/a905ce88736c4c518b72d5adf4544e8f, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/7d986465ea094ff19142a57d7b6b8198, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/938a72eaff8848d680dd98c120f2c359, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/7d8bc2b45817426f9a8d4bff092c1e6b] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp, totalSize=47.0 K 2024-11-14T06:18:39,815 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/673cd1e90d224b348e6b7780ced448fa, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/4d778e54da8c4967bc3d2ee1546d2145, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/eeed945c9aa543a585c59b924ee68eaa, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/193507e26d9d44ec9e0fde2b26b73e92] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp, totalSize=121.0 K 2024-11-14T06:18:39,815 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:39,815 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. files: [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/673cd1e90d224b348e6b7780ced448fa, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/4d778e54da8c4967bc3d2ee1546d2145, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/eeed945c9aa543a585c59b924ee68eaa, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/193507e26d9d44ec9e0fde2b26b73e92] 2024-11-14T06:18:39,816 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting a905ce88736c4c518b72d5adf4544e8f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731565114418 2024-11-14T06:18:39,817 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 673cd1e90d224b348e6b7780ced448fa, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731565114418 2024-11-14T06:18:39,817 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d986465ea094ff19142a57d7b6b8198, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731565115550 2024-11-14T06:18:39,817 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d778e54da8c4967bc3d2ee1546d2145, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731565115550 2024-11-14T06:18:39,817 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 938a72eaff8848d680dd98c120f2c359, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731565116616 2024-11-14T06:18:39,817 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting eeed945c9aa543a585c59b924ee68eaa, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731565116616 2024-11-14T06:18:39,817 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d8bc2b45817426f9a8d4bff092c1e6b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731565117779 2024-11-14T06:18:39,817 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 193507e26d9d44ec9e0fde2b26b73e92, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731565117779 2024-11-14T06:18:39,823 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d062eac452eae155317450043cfef448] 2024-11-14T06:18:39,825 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d062eac452eae155317450043cfef448#B#compaction#495 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:18:39,825 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/b3a4b1a79a79406791ca25d5c25781c2 is 50, key is test_row_0/B:col10/1731565117779/Put/seqid=0 2024-11-14T06:18:39,829 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411144ae8fc4cb2764f65a79e5075a26f4e5b_d062eac452eae155317450043cfef448 store=[table=TestAcidGuarantees family=A region=d062eac452eae155317450043cfef448] 2024-11-14T06:18:39,831 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411144ae8fc4cb2764f65a79e5075a26f4e5b_d062eac452eae155317450043cfef448, store=[table=TestAcidGuarantees family=A region=d062eac452eae155317450043cfef448] 2024-11-14T06:18:39,832 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411144ae8fc4cb2764f65a79e5075a26f4e5b_d062eac452eae155317450043cfef448 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d062eac452eae155317450043cfef448] 2024-11-14T06:18:39,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-14T06:18:39,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:39,845 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing d062eac452eae155317450043cfef448 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-14T06:18:39,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=A 2024-11-14T06:18:39,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:39,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=B 2024-11-14T06:18:39,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:39,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=C 2024-11-14T06:18:39,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:39,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742406_1582 (size=4469) 2024-11-14T06:18:39,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111435116cc3b5284af282c8166b0eb9af57_d062eac452eae155317450043cfef448 is 50, key is test_row_0/A:col10/1731565118942/Put/seqid=0 2024-11-14T06:18:39,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742405_1581 (size=12241) 2024-11-14T06:18:39,881 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/b3a4b1a79a79406791ca25d5c25781c2 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/b3a4b1a79a79406791ca25d5c25781c2 2024-11-14T06:18:39,889 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d062eac452eae155317450043cfef448/B of d062eac452eae155317450043cfef448 into b3a4b1a79a79406791ca25d5c25781c2(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:18:39,889 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d062eac452eae155317450043cfef448: 2024-11-14T06:18:39,889 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., storeName=d062eac452eae155317450043cfef448/B, priority=12, startTime=1731565119814; duration=0sec 2024-11-14T06:18:39,889 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:18:39,889 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d062eac452eae155317450043cfef448:B 2024-11-14T06:18:39,889 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T06:18:39,891 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T06:18:39,891 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): d062eac452eae155317450043cfef448/C is initiating minor compaction (all files) 2024-11-14T06:18:39,891 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d062eac452eae155317450043cfef448/C in TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:39,891 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/fbbae768cd9a49f3a6d2a88189c1c206, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/6c5fd27fa1e3463e8bc8e4d96dd09edb, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/9c81ad76ee1c4f15a191540de38bea88, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/0c2dc43b929a4bf088df1ca94482af1d] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp, totalSize=47.0 K 2024-11-14T06:18:39,891 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting fbbae768cd9a49f3a6d2a88189c1c206, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731565114418 2024-11-14T06:18:39,891 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c5fd27fa1e3463e8bc8e4d96dd09edb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731565115550 2024-11-14T06:18:39,892 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c81ad76ee1c4f15a191540de38bea88, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731565116616 2024-11-14T06:18:39,892 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c2dc43b929a4bf088df1ca94482af1d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731565117779 2024-11-14T06:18:39,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742407_1583 (size=12204) 2024-11-14T06:18:39,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:18:39,898 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111435116cc3b5284af282c8166b0eb9af57_d062eac452eae155317450043cfef448 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111435116cc3b5284af282c8166b0eb9af57_d062eac452eae155317450043cfef448 2024-11-14T06:18:39,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/47256b6c7b5c4a6495b64179663f3560, store: [table=TestAcidGuarantees family=A region=d062eac452eae155317450043cfef448] 2024-11-14T06:18:39,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/47256b6c7b5c4a6495b64179663f3560 is 175, key is test_row_0/A:col10/1731565118942/Put/seqid=0 2024-11-14T06:18:39,902 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d062eac452eae155317450043cfef448#C#compaction#498 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:18:39,902 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/21bded997e4b436cb05d691f7876e983 is 50, key is test_row_0/C:col10/1731565117779/Put/seqid=0 2024-11-14T06:18:39,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742408_1584 (size=31005) 2024-11-14T06:18:39,918 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=130, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/47256b6c7b5c4a6495b64179663f3560 2024-11-14T06:18:39,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742409_1585 (size=12241) 2024-11-14T06:18:39,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/410ce9360959481cab7f0dba9133cf9f is 50, key is test_row_0/B:col10/1731565118942/Put/seqid=0 2024-11-14T06:18:39,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742410_1586 (size=12051) 2024-11-14T06:18:40,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-14T06:18:40,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on d062eac452eae155317450043cfef448 2024-11-14T06:18:40,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. as already flushing 2024-11-14T06:18:40,119 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731565120097, max timestamp=1731565120116Num uniques 859; 2024-11-14T06:18:40,119 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T06:18:40,119 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731565120097, max timestamp=1731565120116Num uniques 859; 2024-11-14T06:18:40,119 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731565120097, max timestamp=1731565120116Num uniques 859; 2024-11-14T06:18:40,119 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T06:18:40,119 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T06:18:40,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:40,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52978 deadline: 1731565180132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:40,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:40,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:40,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:40,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:40,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565180137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:40,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:40,148 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:40,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:40,148 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:40,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565180137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:40,148 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:40,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565180138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:40,148 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:40,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:40,148 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:40,148 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:40,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565180140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:40,150 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:40,150 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:40,150 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:40,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:40,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:40,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:40,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:40,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52978 deadline: 1731565180252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:40,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:40,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:40,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:40,256 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d062eac452eae155317450043cfef448#A#compaction#496 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:18:40,257 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/0ad81e870c754eea95abcae13621ee9c is 175, key is test_row_0/A:col10/1731565117779/Put/seqid=0 2024-11-14T06:18:40,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:40,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565180257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:40,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:40,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565180257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:40,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:40,260 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:40,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565180257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:40,260 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:40,260 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:40,260 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:40,260 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:40,260 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:40,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:40,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565180258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:40,261 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:40,261 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:40,261 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:40,263 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:40,263 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:40,263 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:40,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742411_1587 (size=31195) 2024-11-14T06:18:40,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-14T06:18:40,337 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/21bded997e4b436cb05d691f7876e983 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/21bded997e4b436cb05d691f7876e983 2024-11-14T06:18:40,341 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d062eac452eae155317450043cfef448/C of d062eac452eae155317450043cfef448 into 21bded997e4b436cb05d691f7876e983(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:18:40,341 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d062eac452eae155317450043cfef448: 2024-11-14T06:18:40,341 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., storeName=d062eac452eae155317450043cfef448/C, priority=12, startTime=1731565119814; duration=0sec 2024-11-14T06:18:40,341 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:40,341 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d062eac452eae155317450043cfef448:C 2024-11-14T06:18:40,346 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/410ce9360959481cab7f0dba9133cf9f 2024-11-14T06:18:40,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/d703950c626f47a2952f0220096837f4 is 50, key is test_row_0/C:col10/1731565118942/Put/seqid=0 2024-11-14T06:18:40,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742412_1588 (size=12051) 2024-11-14T06:18:40,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:40,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52978 deadline: 1731565180457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:40,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:40,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:40,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:40,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:40,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565180467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:40,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:40,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:40,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:40,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:40,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565180467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:40,471 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:40,471 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:40,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:40,471 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:40,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565180467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:40,471 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:40,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:40,471 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:40,471 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:40,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565180467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:40,472 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:40,472 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:40,472 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:40,678 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/0ad81e870c754eea95abcae13621ee9c as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/0ad81e870c754eea95abcae13621ee9c 2024-11-14T06:18:40,684 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d062eac452eae155317450043cfef448/A of d062eac452eae155317450043cfef448 into 0ad81e870c754eea95abcae13621ee9c(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:18:40,684 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d062eac452eae155317450043cfef448: 2024-11-14T06:18:40,685 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., storeName=d062eac452eae155317450043cfef448/A, priority=12, startTime=1731565119814; duration=0sec 2024-11-14T06:18:40,690 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:40,690 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d062eac452eae155317450043cfef448:A 2024-11-14T06:18:40,755 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/d703950c626f47a2952f0220096837f4 2024-11-14T06:18:40,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/47256b6c7b5c4a6495b64179663f3560 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/47256b6c7b5c4a6495b64179663f3560 2024-11-14T06:18:40,767 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/47256b6c7b5c4a6495b64179663f3560, entries=150, sequenceid=130, filesize=30.3 K 2024-11-14T06:18:40,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/410ce9360959481cab7f0dba9133cf9f as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/410ce9360959481cab7f0dba9133cf9f 2024-11-14T06:18:40,774 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/410ce9360959481cab7f0dba9133cf9f, entries=150, sequenceid=130, filesize=11.8 K 2024-11-14T06:18:40,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:40,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52978 deadline: 1731565180771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:40,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:40,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:40,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:40,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/d703950c626f47a2952f0220096837f4 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/d703950c626f47a2952f0220096837f4 2024-11-14T06:18:40,781 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/d703950c626f47a2952f0220096837f4, entries=150, sequenceid=130, filesize=11.8 K 2024-11-14T06:18:40,782 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for d062eac452eae155317450043cfef448 in 937ms, sequenceid=130, compaction requested=false 2024-11-14T06:18:40,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for d062eac452eae155317450043cfef448: 2024-11-14T06:18:40,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:40,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-14T06:18:40,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-14T06:18:40,787 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-14T06:18:40,787 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0910 sec 2024-11-14T06:18:40,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on d062eac452eae155317450043cfef448 2024-11-14T06:18:40,789 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d062eac452eae155317450043cfef448 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-14T06:18:40,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=A 2024-11-14T06:18:40,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:18:40,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=B 2024-11-14T06:18:40,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:18:40,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=C 2024-11-14T06:18:40,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:18:40,800 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.1000 sec 2024-11-14T06:18:40,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411143668c70ea87b4a5e952ce32e2f77d2f5_d062eac452eae155317450043cfef448 is 50, key is test_row_0/A:col10/1731565120138/Put/seqid=0 2024-11-14T06:18:40,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:40,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565180801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:40,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:40,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:40,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:40,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:40,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565180805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:40,814 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:40,814 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:40,814 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:40,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:40,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565180805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:40,818 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:40,819 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:40,819 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:40,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:40,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565180813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:40,820 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:40,820 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:40,821 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:40,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-14T06:18:40,827 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:18:40,828 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T06:18:40,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-14T06:18:40,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-14T06:18:40,830 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T06:18:40,830 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T06:18:40,830 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T06:18:40,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742413_1589 (size=17284) 2024-11-14T06:18:40,845 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:18:40,852 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411143668c70ea87b4a5e952ce32e2f77d2f5_d062eac452eae155317450043cfef448 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143668c70ea87b4a5e952ce32e2f77d2f5_d062eac452eae155317450043cfef448 2024-11-14T06:18:40,853 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/1565e4c4c54447f59813b4913e6e261a, store: [table=TestAcidGuarantees family=A region=d062eac452eae155317450043cfef448] 2024-11-14T06:18:40,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/1565e4c4c54447f59813b4913e6e261a is 175, key is test_row_0/A:col10/1731565120138/Put/seqid=0 2024-11-14T06:18:40,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742414_1590 (size=48389) 2024-11-14T06:18:40,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:40,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565180917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:40,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:40,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565180918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:40,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:40,924 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:40,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:40,924 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:40,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:40,924 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:40,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:40,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565180927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:40,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:40,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565180927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:40,933 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:40,933 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:40,933 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:40,934 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:40,934 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:40,934 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:40,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-14T06:18:40,983 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-14T06:18:40,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:40,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. as already flushing 2024-11-14T06:18:40,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:40,984 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:40,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:40,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:41,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:41,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565181127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:41,132 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:41,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:41,132 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:41,132 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:41,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565181127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:41,132 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:41,132 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:41,132 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:41,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-14T06:18:41,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:41,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. as already flushing 2024-11-14T06:18:41,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:41,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:41,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:41,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:41,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:41,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565181138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:41,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:41,142 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:41,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565181138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:41,142 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:41,142 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:41,143 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:41,143 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:41,143 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:41,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-14T06:18:41,280 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=170, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/1565e4c4c54447f59813b4913e6e261a 2024-11-14T06:18:41,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/ff9d569439764f9d8f2302f69f1310dd is 50, key is test_row_0/B:col10/1731565120138/Put/seqid=0 2024-11-14T06:18:41,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742415_1591 (size=12151) 2024-11-14T06:18:41,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-14T06:18:41,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:41,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. as already flushing 2024-11-14T06:18:41,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:41,289 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/ff9d569439764f9d8f2302f69f1310dd 2024-11-14T06:18:41,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:41,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:41,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:41,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:41,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52978 deadline: 1731565181287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:41,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:41,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:41,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:41,296 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/889599a6a8f94ca499876f46c3116e64 is 50, key is test_row_0/C:col10/1731565120138/Put/seqid=0 2024-11-14T06:18:41,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742416_1592 (size=12151) 2024-11-14T06:18:41,301 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/889599a6a8f94ca499876f46c3116e64 2024-11-14T06:18:41,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/1565e4c4c54447f59813b4913e6e261a as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/1565e4c4c54447f59813b4913e6e261a 2024-11-14T06:18:41,308 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/1565e4c4c54447f59813b4913e6e261a, entries=250, sequenceid=170, filesize=47.3 K 2024-11-14T06:18:41,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/ff9d569439764f9d8f2302f69f1310dd as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/ff9d569439764f9d8f2302f69f1310dd 2024-11-14T06:18:41,311 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/ff9d569439764f9d8f2302f69f1310dd, entries=150, sequenceid=170, filesize=11.9 K 2024-11-14T06:18:41,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/889599a6a8f94ca499876f46c3116e64 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/889599a6a8f94ca499876f46c3116e64 2024-11-14T06:18:41,314 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/889599a6a8f94ca499876f46c3116e64, entries=150, sequenceid=170, filesize=11.9 K 2024-11-14T06:18:41,314 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for d062eac452eae155317450043cfef448 in 525ms, sequenceid=170, compaction requested=true 2024-11-14T06:18:41,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d062eac452eae155317450043cfef448: 2024-11-14T06:18:41,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d062eac452eae155317450043cfef448:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T06:18:41,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:41,315 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:18:41,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d062eac452eae155317450043cfef448:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T06:18:41,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:41,315 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:18:41,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d062eac452eae155317450043cfef448:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T06:18:41,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:18:41,315 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36443 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:18:41,315 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): d062eac452eae155317450043cfef448/B is initiating minor compaction (all files) 2024-11-14T06:18:41,316 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d062eac452eae155317450043cfef448/B in TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:41,316 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/b3a4b1a79a79406791ca25d5c25781c2, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/410ce9360959481cab7f0dba9133cf9f, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/ff9d569439764f9d8f2302f69f1310dd] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp, totalSize=35.6 K 2024-11-14T06:18:41,316 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting b3a4b1a79a79406791ca25d5c25781c2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731565117779 2024-11-14T06:18:41,316 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110589 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:18:41,316 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): d062eac452eae155317450043cfef448/A is initiating minor compaction (all files) 2024-11-14T06:18:41,316 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d062eac452eae155317450043cfef448/A in TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:41,316 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/0ad81e870c754eea95abcae13621ee9c, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/47256b6c7b5c4a6495b64179663f3560, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/1565e4c4c54447f59813b4913e6e261a] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp, totalSize=108.0 K 2024-11-14T06:18:41,316 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 410ce9360959481cab7f0dba9133cf9f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731565118941 2024-11-14T06:18:41,316 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:41,316 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. files: [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/0ad81e870c754eea95abcae13621ee9c, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/47256b6c7b5c4a6495b64179663f3560, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/1565e4c4c54447f59813b4913e6e261a] 2024-11-14T06:18:41,316 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting ff9d569439764f9d8f2302f69f1310dd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731565120136 2024-11-14T06:18:41,317 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ad81e870c754eea95abcae13621ee9c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731565117779 2024-11-14T06:18:41,317 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47256b6c7b5c4a6495b64179663f3560, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731565118941 2024-11-14T06:18:41,317 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1565e4c4c54447f59813b4913e6e261a, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731565120136 2024-11-14T06:18:41,322 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d062eac452eae155317450043cfef448] 2024-11-14T06:18:41,322 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d062eac452eae155317450043cfef448#B#compaction#504 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:18:41,322 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/67d4a97def144407971d9e69d5302ed5 is 50, key is test_row_0/B:col10/1731565120138/Put/seqid=0 2024-11-14T06:18:41,323 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111411339631b97f4fe798f74df8a76b047e_d062eac452eae155317450043cfef448 store=[table=TestAcidGuarantees family=A region=d062eac452eae155317450043cfef448] 2024-11-14T06:18:41,325 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111411339631b97f4fe798f74df8a76b047e_d062eac452eae155317450043cfef448, store=[table=TestAcidGuarantees family=A region=d062eac452eae155317450043cfef448] 2024-11-14T06:18:41,325 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111411339631b97f4fe798f74df8a76b047e_d062eac452eae155317450043cfef448 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d062eac452eae155317450043cfef448] 2024-11-14T06:18:41,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742417_1593 (size=12493) 2024-11-14T06:18:41,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742418_1594 (size=4469) 2024-11-14T06:18:41,362 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d062eac452eae155317450043cfef448#A#compaction#505 average throughput is 0.61 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:18:41,362 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/aa0321ef1598490e989b079f14c4c0b7 is 175, key is test_row_0/A:col10/1731565120138/Put/seqid=0 2024-11-14T06:18:41,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742419_1595 (size=31447) 2024-11-14T06:18:41,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on d062eac452eae155317450043cfef448 2024-11-14T06:18:41,441 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d062eac452eae155317450043cfef448 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T06:18:41,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=A 2024-11-14T06:18:41,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:41,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=B 2024-11-14T06:18:41,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:41,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=C 2024-11-14T06:18:41,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:41,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-14T06:18:41,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:41,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. as already flushing 2024-11-14T06:18:41,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:41,442 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:41,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:41,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:41,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114ac75ebbf3d7345ddb90c7b6af12c44a9_d062eac452eae155317450043cfef448 is 50, key is test_row_0/A:col10/1731565120800/Put/seqid=0 2024-11-14T06:18:41,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-14T06:18:41,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742420_1596 (size=12304) 2024-11-14T06:18:41,466 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:18:41,472 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114ac75ebbf3d7345ddb90c7b6af12c44a9_d062eac452eae155317450043cfef448 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ac75ebbf3d7345ddb90c7b6af12c44a9_d062eac452eae155317450043cfef448 2024-11-14T06:18:41,474 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/cf7c6618f20e45969f905c013f13850d, store: [table=TestAcidGuarantees family=A region=d062eac452eae155317450043cfef448] 2024-11-14T06:18:41,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/cf7c6618f20e45969f905c013f13850d is 175, key is test_row_0/A:col10/1731565120800/Put/seqid=0 2024-11-14T06:18:41,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742421_1597 (size=31105) 2024-11-14T06:18:41,480 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=180, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/cf7c6618f20e45969f905c013f13850d 2024-11-14T06:18:41,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:41,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:41,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565181477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:41,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565181478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:41,485 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:41,485 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:41,485 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:41,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731565121465, max timestamp=1731565121482Num uniques 859; 2024-11-14T06:18:41,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T06:18:41,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731565121465, max timestamp=1731565121482Num uniques 859; 2024-11-14T06:18:41,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T06:18:41,488 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731565121465, max timestamp=1731565121482Num uniques 859; 2024-11-14T06:18:41,488 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T06:18:41,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:41,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:41,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:41,506 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/06ba1b23427942f4a7dede27626d9e1b is 50, key is test_row_0/B:col10/1731565120800/Put/seqid=0 2024-11-14T06:18:41,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742422_1598 (size=12151) 2024-11-14T06:18:41,511 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/06ba1b23427942f4a7dede27626d9e1b 2024-11-14T06:18:41,527 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/5e7b5dd5dcba43deab711e1bbc99d3dc is 50, key is test_row_0/C:col10/1731565120800/Put/seqid=0 2024-11-14T06:18:41,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742423_1599 (size=12151) 2024-11-14T06:18:41,565 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/5e7b5dd5dcba43deab711e1bbc99d3dc 2024-11-14T06:18:41,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/cf7c6618f20e45969f905c013f13850d as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/cf7c6618f20e45969f905c013f13850d 2024-11-14T06:18:41,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/cf7c6618f20e45969f905c013f13850d, entries=150, sequenceid=180, filesize=30.4 K 2024-11-14T06:18:41,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/06ba1b23427942f4a7dede27626d9e1b as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/06ba1b23427942f4a7dede27626d9e1b 2024-11-14T06:18:41,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/06ba1b23427942f4a7dede27626d9e1b, entries=150, sequenceid=180, filesize=11.9 K 2024-11-14T06:18:41,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:41,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565181570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:41,579 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:41,579 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:41,579 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:41,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/5e7b5dd5dcba43deab711e1bbc99d3dc as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/5e7b5dd5dcba43deab711e1bbc99d3dc 2024-11-14T06:18:41,583 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/5e7b5dd5dcba43deab711e1bbc99d3dc, entries=150, sequenceid=180, filesize=11.9 K 2024-11-14T06:18:41,583 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for d062eac452eae155317450043cfef448 in 142ms, sequenceid=180, compaction requested=true 2024-11-14T06:18:41,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d062eac452eae155317450043cfef448: 2024-11-14T06:18:41,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d062eac452eae155317450043cfef448:A, priority=-2147483648, current under compaction store size is 3 2024-11-14T06:18:41,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T06:18:41,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d062eac452eae155317450043cfef448:B, priority=-2147483648, current under compaction store size is 3 2024-11-14T06:18:41,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-14T06:18:41,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d062eac452eae155317450043cfef448:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T06:18:41,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-14T06:18:41,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on d062eac452eae155317450043cfef448 2024-11-14T06:18:41,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731565121566, max timestamp=1731565121585Num uniques 1018; 2024-11-14T06:18:41,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-14T06:18:41,590 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731565121566, max timestamp=1731565121585Num uniques 1018; 2024-11-14T06:18:41,590 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731565121566, max timestamp=1731565121585Num uniques 1018; 2024-11-14T06:18:41,590 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-14T06:18:41,590 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-14T06:18:41,590 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d062eac452eae155317450043cfef448 3/3 column families, dataSize=26.84 KB heapSize=65.39 KB 2024-11-14T06:18:41,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=A 2024-11-14T06:18:41,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:18:41,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=B 2024-11-14T06:18:41,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:18:41,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=C 2024-11-14T06:18:41,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:18:41,597 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-14T06:18:41,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:41,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. as already flushing 2024-11-14T06:18:41,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:41,597 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:41,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:41,598 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111449307b894d594f4080b4ea4e968e117e_d062eac452eae155317450043cfef448 is 50, key is test_row_0/A:col10/1731565121566/Put/seqid=0 2024-11-14T06:18:41,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:41,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742424_1600 (size=12304) 2024-11-14T06:18:41,639 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731565121616, max timestamp=1731565121637Num uniques 859; 2024-11-14T06:18:41,639 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T06:18:41,639 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731565121616, max timestamp=1731565121637Num uniques 859; 2024-11-14T06:18:41,639 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T06:18:41,640 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731565121616, max timestamp=1731565121637Num uniques 859; 2024-11-14T06:18:41,640 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T06:18:41,683 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731565121660, max timestamp=1731565121679Num uniques 1068; 2024-11-14T06:18:41,683 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-14T06:18:41,683 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731565121660, max timestamp=1731565121679Num uniques 1068; 2024-11-14T06:18:41,683 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-14T06:18:41,684 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731565121660, max timestamp=1731565121679Num uniques 1068; 2024-11-14T06:18:41,684 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-14T06:18:41,716 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731565121704, max timestamp=1731565121712Num uniques 1027; 2024-11-14T06:18:41,716 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-14T06:18:41,716 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731565121704, max timestamp=1731565121712Num uniques 1027; 2024-11-14T06:18:41,716 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-14T06:18:41,717 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731565121704, max timestamp=1731565121712Num uniques 1027; 2024-11-14T06:18:41,717 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-14T06:18:41,751 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-14T06:18:41,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:41,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. as already flushing 2024-11-14T06:18:41,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:41,751 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:41,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:41,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:41,757 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/67d4a97def144407971d9e69d5302ed5 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/67d4a97def144407971d9e69d5302ed5 2024-11-14T06:18:41,763 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d062eac452eae155317450043cfef448/B of d062eac452eae155317450043cfef448 into 67d4a97def144407971d9e69d5302ed5(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:18:41,763 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d062eac452eae155317450043cfef448: 2024-11-14T06:18:41,763 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., storeName=d062eac452eae155317450043cfef448/B, priority=13, startTime=1731565121315; duration=0sec 2024-11-14T06:18:41,763 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-14T06:18:41,763 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d062eac452eae155317450043cfef448:B 2024-11-14T06:18:41,763 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 3 compacting, 1 eligible, 16 blocking 2024-11-14T06:18:41,763 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-14T06:18:41,764 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-14T06:18:41,764 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. because compaction request was cancelled 2024-11-14T06:18:41,764 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d062eac452eae155317450043cfef448:A 2024-11-14T06:18:41,764 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T06:18:41,765 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10796, totalHeapSize=24784, min timestamp=1731565121742, max timestamp=1731565121761Num uniques 1086; 2024-11-14T06:18:41,766 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49726, new segment data size=10796, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-14T06:18:41,766 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48594 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T06:18:41,766 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): d062eac452eae155317450043cfef448/C is initiating minor compaction (all files) 2024-11-14T06:18:41,766 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d062eac452eae155317450043cfef448/C in TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:41,766 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/21bded997e4b436cb05d691f7876e983, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/d703950c626f47a2952f0220096837f4, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/889599a6a8f94ca499876f46c3116e64, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/5e7b5dd5dcba43deab711e1bbc99d3dc] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp, totalSize=47.5 K 2024-11-14T06:18:41,767 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 21bded997e4b436cb05d691f7876e983, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731565117779 2024-11-14T06:18:41,767 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting d703950c626f47a2952f0220096837f4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731565118941 2024-11-14T06:18:41,768 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 889599a6a8f94ca499876f46c3116e64, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731565120136 2024-11-14T06:18:41,769 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731565121742, max timestamp=1731565121760Num uniques 1086; 2024-11-14T06:18:41,769 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-14T06:18:41,770 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e7b5dd5dcba43deab711e1bbc99d3dc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731565120800 2024-11-14T06:18:41,771 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/aa0321ef1598490e989b079f14c4c0b7 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/aa0321ef1598490e989b079f14c4c0b7 2024-11-14T06:18:41,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731565121749, max timestamp=1731565121760Num uniques 1086; 2024-11-14T06:18:41,772 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=6870, suffix heap size=140912,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=150 2024-11-14T06:18:41,777 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d062eac452eae155317450043cfef448/A of d062eac452eae155317450043cfef448 into aa0321ef1598490e989b079f14c4c0b7(size=30.7 K), total size for store is 61.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:18:41,777 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d062eac452eae155317450043cfef448: 2024-11-14T06:18:41,777 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d062eac452eae155317450043cfef448#C#compaction#510 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:18:41,777 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., storeName=d062eac452eae155317450043cfef448/A, priority=13, startTime=1731565121315; duration=0sec 2024-11-14T06:18:41,777 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T06:18:41,777 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d062eac452eae155317450043cfef448:A 2024-11-14T06:18:41,777 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d062eac452eae155317450043cfef448:B 2024-11-14T06:18:41,777 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 4 compacting, 0 eligible, 16 blocking 2024-11-14T06:18:41,777 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-14T06:18:41,777 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-14T06:18:41,777 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/7cf09b05e01743eb92024d5f03c8faeb is 50, key is test_row_0/C:col10/1731565120800/Put/seqid=0 2024-11-14T06:18:41,777 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. because compaction request was cancelled 2024-11-14T06:18:41,777 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d062eac452eae155317450043cfef448:C 2024-11-14T06:18:41,777 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-14T06:18:41,779 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-14T06:18:41,779 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-14T06:18:41,779 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. because compaction request was cancelled 2024-11-14T06:18:41,779 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d062eac452eae155317450043cfef448:B 2024-11-14T06:18:41,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742425_1601 (size=12527) 2024-11-14T06:18:41,805 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731565121785, max timestamp=1731565121801Num uniques 1095; 2024-11-14T06:18:41,805 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-14T06:18:41,805 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731565121785, max timestamp=1731565121801Num uniques 1095; 2024-11-14T06:18:41,805 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-14T06:18:41,805 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731565121785, max timestamp=1731565121801Num uniques 1009; 2024-11-14T06:18:41,805 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=8930, suffix heap size=132904,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=195 2024-11-14T06:18:41,839 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731565121829, max timestamp=1731565121836Num uniques 1054; 2024-11-14T06:18:41,839 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731565121829, max timestamp=1731565121836Num uniques 1054; 2024-11-14T06:18:41,839 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-11-14T06:18:41,839 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-11-14T06:18:41,840 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731565121829, max timestamp=1731565121836Num uniques 1054; 2024-11-14T06:18:41,840 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-11-14T06:18:41,872 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731565121853, max timestamp=1731565121868Num uniques 1063; 2024-11-14T06:18:41,872 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731565121853, max timestamp=1731565121868Num uniques 1063; 2024-11-14T06:18:41,872 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-14T06:18:41,872 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-14T06:18:41,872 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731565121853, max timestamp=1731565121868Num uniques 1063; 2024-11-14T06:18:41,872 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-14T06:18:41,902 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731565121891, max timestamp=1731565121899Num uniques 1072; 2024-11-14T06:18:41,902 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731565121891, max timestamp=1731565121899Num uniques 1072; 2024-11-14T06:18:41,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-11-14T06:18:41,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-11-14T06:18:41,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731565121891, max timestamp=1731565121899Num uniques 1072; 2024-11-14T06:18:41,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-11-14T06:18:41,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-14T06:18:41,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:41,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. as already flushing 2024-11-14T06:18:41,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:41,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:41,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:41,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:41,936 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8276, totalHeapSize=19064, min timestamp=1731565121917, max timestamp=1731565121933Num uniques 1031; 2024-11-14T06:18:41,936 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47206, new segment data size=8276, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-11-14T06:18:41,936 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8276, totalHeapSize=19064, min timestamp=1731565121917, max timestamp=1731565121933Num uniques 1031; 2024-11-14T06:18:41,936 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47206, new segment data size=8276, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-11-14T06:18:41,936 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8276, totalHeapSize=19064, min timestamp=1731565121917, max timestamp=1731565121933Num uniques 1031; 2024-11-14T06:18:41,936 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47206, new segment data size=8276, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-11-14T06:18:41,962 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=340, cellSize=15570, totalHeapSize=35600, min timestamp=1731565121946, max timestamp=1731565121959Num uniques 1040; 2024-11-14T06:18:41,962 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=15570, suffix heap size=136128,new segment heap size=35600 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=340 2024-11-14T06:18:41,962 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=340, cellSize=15570, totalHeapSize=35600, min timestamp=1731565121946, max timestamp=1731565121959Num uniques 1040; 2024-11-14T06:18:41,962 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=15570, suffix heap size=136128,new segment heap size=35600 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=340 2024-11-14T06:18:41,962 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=340, cellSize=15570, totalHeapSize=35600, min timestamp=1731565121946, max timestamp=1731565121959Num uniques 1040; 2024-11-14T06:18:41,962 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=15570, suffix heap size=136128,new segment heap size=35600 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=340 2024-11-14T06:18:41,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-14T06:18:41,996 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1731565121976, max timestamp=1731565121992Num uniques 1199; 2024-11-14T06:18:41,997 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=54914, new segment data size=11404, suffix heap size=152664,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1199, new segment cells count=249 2024-11-14T06:18:41,997 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=449, cellSize=20564, totalHeapSize=46936, min timestamp=1731565121976, max timestamp=1731565121992Num uniques 1199; 2024-11-14T06:18:41,997 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=54914, new segment data size=20564, suffix heap size=152664,new segment heap size=46936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1199, new segment cells count=449 2024-11-14T06:18:42,000 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1731565121976, max timestamp=1731565121992Num uniques 1199; 2024-11-14T06:18:42,000 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=54914, new segment data size=11404, suffix heap size=152664,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1199, new segment cells count=249 2024-11-14T06:18:42,014 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:18:42,018 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111449307b894d594f4080b4ea4e968e117e_d062eac452eae155317450043cfef448 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111449307b894d594f4080b4ea4e968e117e_d062eac452eae155317450043cfef448 2024-11-14T06:18:42,020 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/8e7acf0dc14047c78dd1894ef528e31b, store: [table=TestAcidGuarantees family=A region=d062eac452eae155317450043cfef448] 2024-11-14T06:18:42,020 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/8e7acf0dc14047c78dd1894ef528e31b is 175, key is test_row_0/A:col10/1731565121566/Put/seqid=0 2024-11-14T06:18:42,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:42,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565182022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:42,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:42,027 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:42,027 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:42,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565182023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:42,027 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:42,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:42,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:42,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:42,028 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=258, cellSize=11810, totalHeapSize=27072, min timestamp=1731565122015, max timestamp=1731565122025Num uniques 1108; 2024-11-14T06:18:42,028 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=11810, suffix heap size=143200,new segment heap size=27072 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=258 2024-11-14T06:18:42,029 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=258, cellSize=11810, totalHeapSize=27072, min timestamp=1731565122015, max timestamp=1731565122025Num uniques 1308; 2024-11-14T06:18:42,029 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=59900, new segment data size=11810, suffix heap size=164000,new segment heap size=27072 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1308, new segment cells count=258 2024-11-14T06:18:42,031 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=258, cellSize=11810, totalHeapSize=27072, min timestamp=1731565122015, max timestamp=1731565122025Num uniques 1108; 2024-11-14T06:18:42,031 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=11810, suffix heap size=143200,new segment heap size=27072 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=258 2024-11-14T06:18:42,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742426_1602 (size=31105) 2024-11-14T06:18:42,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-14T06:18:42,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:42,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. as already flushing 2024-11-14T06:18:42,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:42,057 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:42,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:42,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:42,087 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1731565122066, max timestamp=1731565122083Num uniques 1117; 2024-11-14T06:18:42,087 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1731565122066, max timestamp=1731565122083Num uniques 1117; 2024-11-14T06:18:42,087 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=51152, new segment data size=9932, suffix heap size=144136,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1117, new segment cells count=217 2024-11-14T06:18:42,087 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=51152, new segment data size=9932, suffix heap size=144136,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1117, new segment cells count=217 2024-11-14T06:18:42,087 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1731565122066, max timestamp=1731565122083Num uniques 1117; 2024-11-14T06:18:42,087 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=51152, new segment data size=9932, suffix heap size=144136,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1117, new segment cells count=217 2024-11-14T06:18:42,141 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=176, cellSize=8056, totalHeapSize=18544, min timestamp=1731565122124, max timestamp=1731565122137Num uniques 1076; 2024-11-14T06:18:42,141 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=8056, suffix heap size=139872,new segment heap size=18544 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=176 2024-11-14T06:18:42,142 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=176, cellSize=8056, totalHeapSize=18544, min timestamp=1731565122124, max timestamp=1731565122137Num uniques 1076; 2024-11-14T06:18:42,142 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=8056, suffix heap size=139872,new segment heap size=18544 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=176 2024-11-14T06:18:42,144 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=176, cellSize=8056, totalHeapSize=18544, min timestamp=1731565122124, max timestamp=1731565122137Num uniques 1076; 2024-11-14T06:18:42,144 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=8056, suffix heap size=139872,new segment heap size=18544 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=176 2024-11-14T06:18:42,171 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=235, cellSize=10760, totalHeapSize=24680, min timestamp=1731565122162, max timestamp=1731565122168Num uniques 1035; 2024-11-14T06:18:42,171 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47400, new segment data size=10760, suffix heap size=135608,new segment heap size=24680 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1035, new segment cells count=235 2024-11-14T06:18:42,175 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731565122162, max timestamp=1731565122168Num uniques 1035; 2024-11-14T06:18:42,175 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731565122162, max timestamp=1731565122168Num uniques 1035; 2024-11-14T06:18:42,175 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47400, new segment data size=6870, suffix heap size=135608,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1035, new segment cells count=150 2024-11-14T06:18:42,175 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47400, new segment data size=6870, suffix heap size=135608,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1035, new segment cells count=150 2024-11-14T06:18:42,188 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/7cf09b05e01743eb92024d5f03c8faeb as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/7cf09b05e01743eb92024d5f03c8faeb 2024-11-14T06:18:42,192 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d062eac452eae155317450043cfef448/C of d062eac452eae155317450043cfef448 into 7cf09b05e01743eb92024d5f03c8faeb(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:18:42,192 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d062eac452eae155317450043cfef448: 2024-11-14T06:18:42,192 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., storeName=d062eac452eae155317450043cfef448/C, priority=12, startTime=1731565121584; duration=0sec 2024-11-14T06:18:42,192 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:42,192 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d062eac452eae155317450043cfef448:C 2024-11-14T06:18:42,204 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=294, cellSize=13464, totalHeapSize=30816, min timestamp=1731565122181, max timestamp=1731565122200Num uniques 1094; 2024-11-14T06:18:42,204 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50104, new segment data size=13464, suffix heap size=141744,new segment heap size=30816 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1094, new segment cells count=294 2024-11-14T06:18:42,204 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=294, cellSize=13464, totalHeapSize=30816, min timestamp=1731565122181, max timestamp=1731565122200Num uniques 1009; 2024-11-14T06:18:42,204 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=13464, suffix heap size=132904,new segment heap size=30816 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=294 2024-11-14T06:18:42,205 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=294, cellSize=13464, totalHeapSize=30816, min timestamp=1731565122181, max timestamp=1731565122200Num uniques 1009; 2024-11-14T06:18:42,205 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=13464, suffix heap size=132904,new segment heap size=30816 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=294 2024-11-14T06:18:42,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-14T06:18:42,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:42,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. as already flushing 2024-11-14T06:18:42,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:42,210 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:42,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:42,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:42,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:42,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565182226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:42,236 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=203, cellSize=9295, totalHeapSize=21352, min timestamp=1731565122214, max timestamp=1731565122233Num uniques 1153; 2024-11-14T06:18:42,236 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52805, new segment data size=9295, suffix heap size=147880,new segment heap size=21352 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1153, new segment cells count=203 2024-11-14T06:18:42,236 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=203, cellSize=9295, totalHeapSize=21352, min timestamp=1731565122214, max timestamp=1731565122233Num uniques 1153; 2024-11-14T06:18:42,236 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52805, new segment data size=9295, suffix heap size=147880,new segment heap size=21352 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1153, new segment cells count=203 2024-11-14T06:18:42,237 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=203, cellSize=9295, totalHeapSize=21352, min timestamp=1731565122214, max timestamp=1731565122233Num uniques 1153; 2024-11-14T06:18:42,237 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=52805, new segment data size=9295, suffix heap size=147880,new segment heap size=21352 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1153, new segment cells count=203 2024-11-14T06:18:42,237 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:42,237 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:42,237 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:42,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=212, cellSize=9702, totalHeapSize=22288, min timestamp=1731565122257, max timestamp=1731565122270Num uniques 1062; 2024-11-14T06:18:42,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48632, new segment data size=9702, suffix heap size=138416,new segment heap size=22288 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1062, new segment cells count=212 2024-11-14T06:18:42,275 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731565122257, max timestamp=1731565122270Num uniques 1062; 2024-11-14T06:18:42,275 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48632, new segment data size=6870, suffix heap size=138416,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1062, new segment cells count=150 2024-11-14T06:18:42,275 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731565122257, max timestamp=1731565122270Num uniques 1062; 2024-11-14T06:18:42,275 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48632, new segment data size=6870, suffix heap size=138416,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1062, new segment cells count=150 2024-11-14T06:18:42,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=221, cellSize=10116, totalHeapSize=23224, min timestamp=1731565122296, max timestamp=1731565122311Num uniques 1009; 2024-11-14T06:18:42,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=10116, suffix heap size=132904,new segment heap size=23224 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=221 2024-11-14T06:18:42,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=221, cellSize=10116, totalHeapSize=23224, min timestamp=1731565122296, max timestamp=1731565122311Num uniques 1009; 2024-11-14T06:18:42,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=10116, suffix heap size=132904,new segment heap size=23224 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=221 2024-11-14T06:18:42,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=221, cellSize=10116, totalHeapSize=23224, min timestamp=1731565122296, max timestamp=1731565122311Num uniques 1071; 2024-11-14T06:18:42,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49046, new segment data size=10116, suffix heap size=139352,new segment heap size=23224 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1071, new segment cells count=221 2024-11-14T06:18:42,348 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=230, cellSize=10530, totalHeapSize=24160, min timestamp=1731565122327, max timestamp=1731565122342Num uniques 1080; 2024-11-14T06:18:42,348 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=230, cellSize=10530, totalHeapSize=24160, min timestamp=1731565122327, max timestamp=1731565122342Num uniques 1080; 2024-11-14T06:18:42,348 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49460, new segment data size=10530, suffix heap size=140288,new segment heap size=24160 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1080, new segment cells count=230 2024-11-14T06:18:42,348 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49460, new segment data size=10530, suffix heap size=140288,new segment heap size=24160 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1080, new segment cells count=230 2024-11-14T06:18:42,349 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=230, cellSize=10530, totalHeapSize=24160, min timestamp=1731565122327, max timestamp=1731565122345Num uniques 1080; 2024-11-14T06:18:42,349 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49460, new segment data size=10530, suffix heap size=140288,new segment heap size=24160 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1080, new segment cells count=230 2024-11-14T06:18:42,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-14T06:18:42,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:42,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. as already flushing 2024-11-14T06:18:42,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:42,363 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:42,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:42,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:42,379 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=239, cellSize=10944, totalHeapSize=25096, min timestamp=1731565122369, max timestamp=1731565122375Num uniques 1089; 2024-11-14T06:18:42,379 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49874, new segment data size=10944, suffix heap size=141224,new segment heap size=25096 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1089, new segment cells count=239 2024-11-14T06:18:42,379 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=239, cellSize=10944, totalHeapSize=25096, min timestamp=1731565122369, max timestamp=1731565122375Num uniques 1089; 2024-11-14T06:18:42,379 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49874, new segment data size=10944, suffix heap size=141224,new segment heap size=25096 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1089, new segment cells count=239 2024-11-14T06:18:42,380 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=239, cellSize=10944, totalHeapSize=25096, min timestamp=1731565122369, max timestamp=1731565122375Num uniques 1089; 2024-11-14T06:18:42,380 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49874, new segment data size=10944, suffix heap size=141224,new segment heap size=25096 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1089, new segment cells count=239 2024-11-14T06:18:42,405 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=348, cellSize=15938, totalHeapSize=36432, min timestamp=1731565122389, max timestamp=1731565122401Num uniques 1098; 2024-11-14T06:18:42,405 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50288, new segment data size=15938, suffix heap size=142160,new segment heap size=36432 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1098, new segment cells count=348 2024-11-14T06:18:42,405 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=348, cellSize=15938, totalHeapSize=36432, min timestamp=1731565122389, max timestamp=1731565122401Num uniques 1098; 2024-11-14T06:18:42,405 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50288, new segment data size=15938, suffix heap size=142160,new segment heap size=36432 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1098, new segment cells count=348 2024-11-14T06:18:42,405 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=348, cellSize=15938, totalHeapSize=36432, min timestamp=1731565122389, max timestamp=1731565122401Num uniques 1098; 2024-11-14T06:18:42,405 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50288, new segment data size=15938, suffix heap size=142160,new segment heap size=36432 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1098, new segment cells count=348 2024-11-14T06:18:42,437 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=218, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/8e7acf0dc14047c78dd1894ef528e31b 2024-11-14T06:18:42,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:42,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565182431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:42,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:42,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565182434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:42,442 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:42,442 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:42,442 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:42,442 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:42,442 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:42,442 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:42,444 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/e450e0ca67304e95b7df979cf7ad9a7b is 50, key is test_row_0/B:col10/1731565121566/Put/seqid=0 2024-11-14T06:18:42,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:42,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565182438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:42,448 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:42,449 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:42,449 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:42,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742427_1603 (size=12151) 2024-11-14T06:18:42,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:42,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565182442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:42,455 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=157, cellSize=7185, totalHeapSize=16568, min timestamp=1731565122426, max timestamp=1731565122439Num uniques 1207; 2024-11-14T06:18:42,455 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=55275, new segment data size=7185, suffix heap size=153496,new segment heap size=16568 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1207, new segment cells count=157 2024-11-14T06:18:42,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:42,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:42,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:42,457 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=157, cellSize=7185, totalHeapSize=16568, min timestamp=1731565122426, max timestamp=1731565122439Num uniques 1207; 2024-11-14T06:18:42,457 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=55275, new segment data size=7185, suffix heap size=153496,new segment heap size=16568 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1207, new segment cells count=157 2024-11-14T06:18:42,460 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731565122426, max timestamp=1731565122439Num uniques 1207; 2024-11-14T06:18:42,460 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=55275, new segment data size=6870, suffix heap size=153496,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1207, new segment cells count=150 2024-11-14T06:18:42,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-14T06:18:42,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:42,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. as already flushing 2024-11-14T06:18:42,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:42,516 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:42,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:42,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:42,548 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=166, cellSize=7596, totalHeapSize=17504, min timestamp=1731565122529, max timestamp=1731565122544Num uniques 1016; 2024-11-14T06:18:42,548 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=166, cellSize=7596, totalHeapSize=17504, min timestamp=1731565122529, max timestamp=1731565122544Num uniques 1016; 2024-11-14T06:18:42,548 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46526, new segment data size=7596, suffix heap size=133632,new segment heap size=17504 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1016, new segment cells count=166 2024-11-14T06:18:42,548 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46526, new segment data size=7596, suffix heap size=133632,new segment heap size=17504 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1016, new segment cells count=166 2024-11-14T06:18:42,548 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=166, cellSize=7596, totalHeapSize=17504, min timestamp=1731565122529, max timestamp=1731565122544Num uniques 1009; 2024-11-14T06:18:42,548 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46211, new segment data size=7596, suffix heap size=132904,new segment heap size=17504 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=166 2024-11-14T06:18:42,576 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=275, cellSize=12588, totalHeapSize=28840, min timestamp=1731565122565, max timestamp=1731565122573Num uniques 1025; 2024-11-14T06:18:42,576 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=275, cellSize=12580, totalHeapSize=28840, min timestamp=1731565122565, max timestamp=1731565122573Num uniques 1025; 2024-11-14T06:18:42,576 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46938, new segment data size=12588, suffix heap size=134568,new segment heap size=28840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1025, new segment cells count=275 2024-11-14T06:18:42,576 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46930, new segment data size=12580, suffix heap size=134568,new segment heap size=28840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1025, new segment cells count=275 2024-11-14T06:18:42,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=275, cellSize=12590, totalHeapSize=28840, min timestamp=1731565122565, max timestamp=1731565122573Num uniques 1025; 2024-11-14T06:18:42,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46940, new segment data size=12590, suffix heap size=134568,new segment heap size=28840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1025, new segment cells count=275 2024-11-14T06:18:42,600 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=284, cellSize=13004, totalHeapSize=29776, min timestamp=1731565122582, max timestamp=1731565122597Num uniques 1134; 2024-11-14T06:18:42,600 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51934, new segment data size=13004, suffix heap size=145904,new segment heap size=29776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1134, new segment cells count=284 2024-11-14T06:18:42,600 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=284, cellSize=13004, totalHeapSize=29776, min timestamp=1731565122582, max timestamp=1731565122597Num uniques 1134; 2024-11-14T06:18:42,600 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=51934, new segment data size=13004, suffix heap size=145904,new segment heap size=29776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1134, new segment cells count=284 2024-11-14T06:18:42,600 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=284, cellSize=13004, totalHeapSize=29776, min timestamp=1731565122582, max timestamp=1731565122597Num uniques 1134; 2024-11-14T06:18:42,600 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=51934, new segment data size=13004, suffix heap size=145904,new segment heap size=29776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1134, new segment cells count=284 2024-11-14T06:18:42,621 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=293, cellSize=13418, totalHeapSize=30712, min timestamp=1731565122610, max timestamp=1731565122617Num uniques 1143; 2024-11-14T06:18:42,621 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=293, cellSize=13418, totalHeapSize=30712, min timestamp=1731565122610, max timestamp=1731565122617Num uniques 1143; 2024-11-14T06:18:42,621 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=52348, new segment data size=13418, suffix heap size=146840,new segment heap size=30712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1143, new segment cells count=293 2024-11-14T06:18:42,621 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52348, new segment data size=13418, suffix heap size=146840,new segment heap size=30712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1143, new segment cells count=293 2024-11-14T06:18:42,623 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=193, cellSize=8838, totalHeapSize=20312, min timestamp=1731565122614, max timestamp=1731565122617Num uniques 1143; 2024-11-14T06:18:42,623 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=52348, new segment data size=8838, suffix heap size=146840,new segment heap size=20312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1143, new segment cells count=193 2024-11-14T06:18:42,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=152, cellSize=6960, totalHeapSize=16048, min timestamp=1731565122639, max timestamp=1731565122642Num uniques 1152; 2024-11-14T06:18:42,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=52760, new segment data size=6960, suffix heap size=147776,new segment heap size=16048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1152, new segment cells count=152 2024-11-14T06:18:42,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=152, cellSize=6960, totalHeapSize=16048, min timestamp=1731565122639, max timestamp=1731565122642Num uniques 1152; 2024-11-14T06:18:42,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=52760, new segment data size=6960, suffix heap size=147776,new segment heap size=16048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1152, new segment cells count=152 2024-11-14T06:18:42,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=152, cellSize=6960, totalHeapSize=16048, min timestamp=1731565122639, max timestamp=1731565122642Num uniques 1052; 2024-11-14T06:18:42,646 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48180, new segment data size=6960, suffix heap size=137376,new segment heap size=16048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1052, new segment cells count=152 2024-11-14T06:18:42,671 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=161, cellSize=7366, totalHeapSize=16984, min timestamp=1731565122661, max timestamp=1731565122668Num uniques 1011; 2024-11-14T06:18:42,671 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46296, new segment data size=7366, suffix heap size=133112,new segment heap size=16984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1011, new segment cells count=161 2024-11-14T06:18:42,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-14T06:18:42,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:42,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. as already flushing 2024-11-14T06:18:42,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:42,672 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:42,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:42,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:42,673 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=161, cellSize=7366, totalHeapSize=16984, min timestamp=1731565122661, max timestamp=1731565122668Num uniques 1011; 2024-11-14T06:18:42,673 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46296, new segment data size=7366, suffix heap size=133112,new segment heap size=16984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1011, new segment cells count=161 2024-11-14T06:18:42,673 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=161, cellSize=7366, totalHeapSize=16984, min timestamp=1731565122661, max timestamp=1731565122668Num uniques 1011; 2024-11-14T06:18:42,673 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46296, new segment data size=7366, suffix heap size=133112,new segment heap size=16984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1011, new segment cells count=161 2024-11-14T06:18:42,695 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=170, cellSize=7780, totalHeapSize=17920, min timestamp=1731565122685, max timestamp=1731565122692Num uniques 1020; 2024-11-14T06:18:42,695 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46710, new segment data size=7780, suffix heap size=134048,new segment heap size=17920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1020, new segment cells count=170 2024-11-14T06:18:42,695 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=220, cellSize=10070, totalHeapSize=23120, min timestamp=1731565122685, max timestamp=1731565122692Num uniques 1020; 2024-11-14T06:18:42,695 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46710, new segment data size=10070, suffix heap size=134048,new segment heap size=23120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1020, new segment cells count=220 2024-11-14T06:18:42,698 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=170, cellSize=7780, totalHeapSize=17920, min timestamp=1731565122685, max timestamp=1731565122692Num uniques 1020; 2024-11-14T06:18:42,698 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46710, new segment data size=7780, suffix heap size=134048,new segment heap size=17920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1020, new segment cells count=170 2024-11-14T06:18:42,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=229, cellSize=10481, totalHeapSize=24056, min timestamp=1731565122708, max timestamp=1731565122716Num uniques 1029; 2024-11-14T06:18:42,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47121, new segment data size=10481, suffix heap size=134984,new segment heap size=24056 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1029, new segment cells count=229 2024-11-14T06:18:42,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=229, cellSize=10484, totalHeapSize=24056, min timestamp=1731565122708, max timestamp=1731565122716Num uniques 1079; 2024-11-14T06:18:42,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49414, new segment data size=10484, suffix heap size=140184,new segment heap size=24056 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1079, new segment cells count=229 2024-11-14T06:18:42,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=229, cellSize=10474, totalHeapSize=24056, min timestamp=1731565122708, max timestamp=1731565122716Num uniques 1029; 2024-11-14T06:18:42,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47114, new segment data size=10474, suffix heap size=134984,new segment heap size=24056 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1029, new segment cells count=229 2024-11-14T06:18:42,743 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=238, cellSize=10898, totalHeapSize=24992, min timestamp=1731565122727, max timestamp=1731565122740Num uniques 1088; 2024-11-14T06:18:42,743 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49828, new segment data size=10898, suffix heap size=141120,new segment heap size=24992 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1088, new segment cells count=238 2024-11-14T06:18:42,744 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=238, cellSize=10898, totalHeapSize=24992, min timestamp=1731565122727, max timestamp=1731565122740Num uniques 1088; 2024-11-14T06:18:42,744 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49828, new segment data size=10898, suffix heap size=141120,new segment heap size=24992 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1088, new segment cells count=238 2024-11-14T06:18:42,744 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=238, cellSize=10898, totalHeapSize=24992, min timestamp=1731565122727, max timestamp=1731565122740Num uniques 1088; 2024-11-14T06:18:42,744 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49828, new segment data size=10898, suffix heap size=141120,new segment heap size=24992 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1088, new segment cells count=238 2024-11-14T06:18:42,770 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=247, cellSize=11312, totalHeapSize=25928, min timestamp=1731565122757, max timestamp=1731565122767Num uniques 1097; 2024-11-14T06:18:42,770 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50242, new segment data size=11312, suffix heap size=142056,new segment heap size=25928 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1097, new segment cells count=247 2024-11-14T06:18:42,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=347, cellSize=15892, totalHeapSize=36328, min timestamp=1731565122756, max timestamp=1731565122767Num uniques 1097; 2024-11-14T06:18:42,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50242, new segment data size=15892, suffix heap size=142056,new segment heap size=36328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1097, new segment cells count=347 2024-11-14T06:18:42,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=247, cellSize=11312, totalHeapSize=25928, min timestamp=1731565122757, max timestamp=1731565122767Num uniques 1097; 2024-11-14T06:18:42,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50242, new segment data size=11312, suffix heap size=142056,new segment heap size=25928 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1097, new segment cells count=247 2024-11-14T06:18:42,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:42,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:42,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565182793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:42,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565182794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:42,800 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:42,800 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:42,800 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:42,802 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=206, cellSize=9430, totalHeapSize=21664, min timestamp=1731565122785, max timestamp=1731565122798Num uniques 1106; 2024-11-14T06:18:42,802 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50650, new segment data size=9430, suffix heap size=142992,new segment heap size=21664 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1106, new segment cells count=206 2024-11-14T06:18:42,802 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=206, cellSize=9430, totalHeapSize=21664, min timestamp=1731565122785, max timestamp=1731565122798Num uniques 1106; 2024-11-14T06:18:42,802 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=206, cellSize=9430, totalHeapSize=21664, min timestamp=1731565122785, max timestamp=1731565122798Num uniques 1206; 2024-11-14T06:18:42,802 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50650, new segment data size=9430, suffix heap size=142992,new segment heap size=21664 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1106, new segment cells count=206 2024-11-14T06:18:42,802 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:42,802 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=55230, new segment data size=9430, suffix heap size=153392,new segment heap size=21664 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1206, new segment cells count=206 2024-11-14T06:18:42,802 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:42,802 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:42,824 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-14T06:18:42,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:42,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. as already flushing 2024-11-14T06:18:42,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:42,824 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:42,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:42,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:42,850 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/e450e0ca67304e95b7df979cf7ad9a7b 2024-11-14T06:18:42,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/00991d7cbaa14366b6b5bd73102565ba is 50, key is test_row_0/C:col10/1731565121566/Put/seqid=0 2024-11-14T06:18:42,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742428_1604 (size=12151) 2024-11-14T06:18:42,875 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=165, cellSize=7550, totalHeapSize=17400, min timestamp=1731565122823, max timestamp=1731565122850Num uniques 1065; 2024-11-14T06:18:42,875 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48770, new segment data size=7550, suffix heap size=138728,new segment heap size=17400 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1065, new segment cells count=165 2024-11-14T06:18:42,875 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=165, cellSize=7545, totalHeapSize=17400, min timestamp=1731565122823, max timestamp=1731565122850Num uniques 1065; 2024-11-14T06:18:42,875 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48765, new segment data size=7545, suffix heap size=138728,new segment heap size=17400 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1065, new segment cells count=165 2024-11-14T06:18:42,875 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=165, cellSize=7547, totalHeapSize=17400, min timestamp=1731565122823, max timestamp=1731565122850Num uniques 1065; 2024-11-14T06:18:42,875 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48767, new segment data size=7547, suffix heap size=138728,new segment heap size=17400 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1065, new segment cells count=165 2024-11-14T06:18:42,911 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=224, cellSize=10254, totalHeapSize=23536, min timestamp=1731565122892, max timestamp=1731565122907Num uniques 1024; 2024-11-14T06:18:42,911 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46894, new segment data size=10254, suffix heap size=134464,new segment heap size=23536 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1024, new segment cells count=224 2024-11-14T06:18:42,911 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=224, cellSize=10254, totalHeapSize=23536, min timestamp=1731565122892, max timestamp=1731565122907Num uniques 1024; 2024-11-14T06:18:42,911 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46894, new segment data size=10254, suffix heap size=134464,new segment heap size=23536 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1024, new segment cells count=224 2024-11-14T06:18:42,911 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=224, cellSize=10254, totalHeapSize=23536, min timestamp=1731565122892, max timestamp=1731565122907Num uniques 1024; 2024-11-14T06:18:42,911 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46894, new segment data size=10254, suffix heap size=134464,new segment heap size=23536 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1024, new segment cells count=224 2024-11-14T06:18:42,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=283, cellSize=12958, totalHeapSize=29672, min timestamp=1731565122928, max timestamp=1731565122940Num uniques 1083; 2024-11-14T06:18:42,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49598, new segment data size=12958, suffix heap size=140600,new segment heap size=29672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1083, new segment cells count=283 2024-11-14T06:18:42,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=283, cellSize=12958, totalHeapSize=29672, min timestamp=1731565122928, max timestamp=1731565122940Num uniques 1083; 2024-11-14T06:18:42,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49598, new segment data size=12958, suffix heap size=140600,new segment heap size=29672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1083, new segment cells count=283 2024-11-14T06:18:42,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=283, cellSize=12958, totalHeapSize=29672, min timestamp=1731565122928, max timestamp=1731565122940Num uniques 1083; 2024-11-14T06:18:42,945 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49598, new segment data size=12958, suffix heap size=140600,new segment heap size=29672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1083, new segment cells count=283 2024-11-14T06:18:42,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-14T06:18:42,978 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-14T06:18:42,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:42,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:42,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. as already flushing 2024-11-14T06:18:42,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:42,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565182971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:42,978 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:42,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:42,979 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:42,979 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:42,979 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:42,981 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=192, cellSize=8792, totalHeapSize=20208, min timestamp=1731565122966, max timestamp=1731565122975Num uniques 1142; 2024-11-14T06:18:42,981 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52302, new segment data size=8792, suffix heap size=146736,new segment heap size=20208 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1142, new segment cells count=192 2024-11-14T06:18:42,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=192, cellSize=8792, totalHeapSize=20208, min timestamp=1731565122966, max timestamp=1731565122975Num uniques 1142; 2024-11-14T06:18:42,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=52302, new segment data size=8792, suffix heap size=146736,new segment heap size=20208 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1142, new segment cells count=192 2024-11-14T06:18:42,983 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=192, cellSize=8792, totalHeapSize=20208, min timestamp=1731565122966, max timestamp=1731565122975Num uniques 1142; 2024-11-14T06:18:42,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:42,983 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52302, new segment data size=8792, suffix heap size=146736,new segment heap size=20208 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1142, new segment cells count=192 2024-11-14T06:18:43,013 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=151, cellSize=6915, totalHeapSize=15944, min timestamp=1731565123002, max timestamp=1731565123010Num uniques 1051; 2024-11-14T06:18:43,013 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=151, cellSize=6915, totalHeapSize=15944, min timestamp=1731565123002, max timestamp=1731565123010Num uniques 1051; 2024-11-14T06:18:43,013 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48135, new segment data size=6915, suffix heap size=137272,new segment heap size=15944 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1051, new segment cells count=151 2024-11-14T06:18:43,013 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48135, new segment data size=6915, suffix heap size=137272,new segment heap size=15944 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1051, new segment cells count=151 2024-11-14T06:18:43,013 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=151, cellSize=6915, totalHeapSize=15944, min timestamp=1731565123002, max timestamp=1731565123010Num uniques 1051; 2024-11-14T06:18:43,013 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48135, new segment data size=6915, suffix heap size=137272,new segment heap size=15944 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1051, new segment cells count=151 2024-11-14T06:18:43,042 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=210, cellSize=9610, totalHeapSize=22080, min timestamp=1731565123021, max timestamp=1731565123039Num uniques 1010; 2024-11-14T06:18:43,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46250, new segment data size=9610, suffix heap size=133008,new segment heap size=22080 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1010, new segment cells count=210 2024-11-14T06:18:43,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=210, cellSize=9610, totalHeapSize=22080, min timestamp=1731565123021, max timestamp=1731565123039Num uniques 1010; 2024-11-14T06:18:43,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46250, new segment data size=9610, suffix heap size=133008,new segment heap size=22080 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1010, new segment cells count=210 2024-11-14T06:18:43,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=210, cellSize=9610, totalHeapSize=22080, min timestamp=1731565123021, max timestamp=1731565123039Num uniques 1010; 2024-11-14T06:18:43,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46250, new segment data size=9610, suffix heap size=133008,new segment heap size=22080 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1010, new segment cells count=210 2024-11-14T06:18:43,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=169, cellSize=7734, totalHeapSize=17816, min timestamp=1731565123064, max timestamp=1731565123075Num uniques 1069; 2024-11-14T06:18:43,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48954, new segment data size=7734, suffix heap size=139144,new segment heap size=17816 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1069, new segment cells count=169 2024-11-14T06:18:43,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=169, cellSize=7734, totalHeapSize=17816, min timestamp=1731565123064, max timestamp=1731565123075Num uniques 1069; 2024-11-14T06:18:43,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=169, cellSize=7734, totalHeapSize=17816, min timestamp=1731565123064, max timestamp=1731565123075Num uniques 1069; 2024-11-14T06:18:43,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48954, new segment data size=7734, suffix heap size=139144,new segment heap size=17816 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1069, new segment cells count=169 2024-11-14T06:18:43,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48954, new segment data size=7734, suffix heap size=139144,new segment heap size=17816 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1069, new segment cells count=169 2024-11-14T06:18:43,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=228, cellSize=10438, totalHeapSize=23952, min timestamp=1731565123092, max timestamp=1731565123100Num uniques 1028; 2024-11-14T06:18:43,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=228, cellSize=10438, totalHeapSize=23952, min timestamp=1731565123092, max timestamp=1731565123100Num uniques 1028; 2024-11-14T06:18:43,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47078, new segment data size=10438, suffix heap size=134880,new segment heap size=23952 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1028, new segment cells count=228 2024-11-14T06:18:43,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47078, new segment data size=10438, suffix heap size=134880,new segment heap size=23952 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1028, new segment cells count=228 2024-11-14T06:18:43,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=228, cellSize=10438, totalHeapSize=23952, min timestamp=1731565123092, max timestamp=1731565123100Num uniques 1028; 2024-11-14T06:18:43,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47078, new segment data size=10438, suffix heap size=134880,new segment heap size=23952 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1028, new segment cells count=228 2024-11-14T06:18:43,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:43,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565183123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:43,129 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:43,129 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:43,129 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:43,129 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=237, cellSize=10852, totalHeapSize=24888, min timestamp=1731565123114, max timestamp=1731565123126Num uniques 1087; 2024-11-14T06:18:43,130 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49782, new segment data size=10852, suffix heap size=141016,new segment heap size=24888 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1087, new segment cells count=237 2024-11-14T06:18:43,130 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=237, cellSize=10852, totalHeapSize=24888, min timestamp=1731565123114, max timestamp=1731565123126Num uniques 1087; 2024-11-14T06:18:43,130 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49782, new segment data size=10852, suffix heap size=141016,new segment heap size=24888 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1087, new segment cells count=237 2024-11-14T06:18:43,130 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=237, cellSize=10852, totalHeapSize=24888, min timestamp=1731565123114, max timestamp=1731565123126Num uniques 1087; 2024-11-14T06:18:43,130 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49782, new segment data size=10852, suffix heap size=141016,new segment heap size=24888 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1087, new segment cells count=237 2024-11-14T06:18:43,135 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-14T06:18:43,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:43,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. as already flushing 2024-11-14T06:18:43,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:43,136 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:43,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:43,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:43,157 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=196, cellSize=8976, totalHeapSize=20624, min timestamp=1731565123139, max timestamp=1731565123154Num uniques 1096; 2024-11-14T06:18:43,157 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=196, cellSize=8976, totalHeapSize=20624, min timestamp=1731565123139, max timestamp=1731565123154Num uniques 1096; 2024-11-14T06:18:43,157 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50196, new segment data size=8976, suffix heap size=141952,new segment heap size=20624 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1096, new segment cells count=196 2024-11-14T06:18:43,157 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50196, new segment data size=8976, suffix heap size=141952,new segment heap size=20624 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1096, new segment cells count=196 2024-11-14T06:18:43,158 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=196, cellSize=8976, totalHeapSize=20624, min timestamp=1731565123139, max timestamp=1731565123154Num uniques 1096; 2024-11-14T06:18:43,158 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50196, new segment data size=8976, suffix heap size=141952,new segment heap size=20624 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1096, new segment cells count=196 2024-11-14T06:18:43,185 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=255, cellSize=11675, totalHeapSize=26760, min timestamp=1731565123172, max timestamp=1731565123182Num uniques 1055; 2024-11-14T06:18:43,185 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48315, new segment data size=11675, suffix heap size=137688,new segment heap size=26760 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1055, new segment cells count=255 2024-11-14T06:18:43,186 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=255, cellSize=11675, totalHeapSize=26760, min timestamp=1731565123172, max timestamp=1731565123182Num uniques 1055; 2024-11-14T06:18:43,186 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48315, new segment data size=11675, suffix heap size=137688,new segment heap size=26760 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1055, new segment cells count=255 2024-11-14T06:18:43,188 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=205, cellSize=9385, totalHeapSize=21560, min timestamp=1731565123172, max timestamp=1731565123182Num uniques 1055; 2024-11-14T06:18:43,188 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48315, new segment data size=9385, suffix heap size=137688,new segment heap size=21560 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1055, new segment cells count=205 2024-11-14T06:18:43,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=214, cellSize=9794, totalHeapSize=22496, min timestamp=1731565123197, max timestamp=1731565123210Num uniques 1114; 2024-11-14T06:18:43,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=51014, new segment data size=9794, suffix heap size=143824,new segment heap size=22496 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1114, new segment cells count=214 2024-11-14T06:18:43,214 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=214, cellSize=9794, totalHeapSize=22496, min timestamp=1731565123197, max timestamp=1731565123210Num uniques 1114; 2024-11-14T06:18:43,214 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51014, new segment data size=9794, suffix heap size=143824,new segment heap size=22496 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1114, new segment cells count=214 2024-11-14T06:18:43,214 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=214, cellSize=9794, totalHeapSize=22496, min timestamp=1731565123197, max timestamp=1731565123210Num uniques 1064; 2024-11-14T06:18:43,214 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48724, new segment data size=9794, suffix heap size=138624,new segment heap size=22496 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1064, new segment cells count=214 2024-11-14T06:18:43,266 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/00991d7cbaa14366b6b5bd73102565ba 2024-11-14T06:18:43,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/8e7acf0dc14047c78dd1894ef528e31b as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/8e7acf0dc14047c78dd1894ef528e31b 2024-11-14T06:18:43,271 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=223, cellSize=10208, totalHeapSize=23432, min timestamp=1731565123259, max timestamp=1731565123267Num uniques 1073; 2024-11-14T06:18:43,271 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49138, new segment data size=10208, suffix heap size=139560,new segment heap size=23432 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1073, new segment cells count=223 2024-11-14T06:18:43,273 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/8e7acf0dc14047c78dd1894ef528e31b, entries=150, sequenceid=218, filesize=30.4 K 2024-11-14T06:18:43,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731565123259, max timestamp=1731565123267Num uniques 1073; 2024-11-14T06:18:43,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49138, new segment data size=6870, suffix heap size=139560,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1073, new segment cells count=150 2024-11-14T06:18:43,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/e450e0ca67304e95b7df979cf7ad9a7b as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/e450e0ca67304e95b7df979cf7ad9a7b 2024-11-14T06:18:43,277 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/e450e0ca67304e95b7df979cf7ad9a7b, entries=150, sequenceid=218, filesize=11.9 K 2024-11-14T06:18:43,277 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731565123259, max timestamp=1731565123267Num uniques 1073; 2024-11-14T06:18:43,278 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef93ef3a83c5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49138, new segment data size=6870, suffix heap size=139560,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1073, new segment cells count=150 2024-11-14T06:18:43,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/00991d7cbaa14366b6b5bd73102565ba as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/00991d7cbaa14366b6b5bd73102565ba 2024-11-14T06:18:43,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/00991d7cbaa14366b6b5bd73102565ba, entries=150, sequenceid=218, filesize=11.9 K 2024-11-14T06:18:43,281 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~65.34 KB/66912, currentSize=73.99 KB/75764 for d062eac452eae155317450043cfef448 in 1691ms, sequenceid=218, compaction requested=true 2024-11-14T06:18:43,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d062eac452eae155317450043cfef448: 2024-11-14T06:18:43,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d062eac452eae155317450043cfef448:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T06:18:43,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:43,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d062eac452eae155317450043cfef448:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T06:18:43,282 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:18:43,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:43,282 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:18:43,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d062eac452eae155317450043cfef448:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T06:18:43,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:18:43,282 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:18:43,282 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): d062eac452eae155317450043cfef448/B is initiating minor compaction (all files) 2024-11-14T06:18:43,282 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d062eac452eae155317450043cfef448/B in TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:43,282 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/67d4a97def144407971d9e69d5302ed5, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/06ba1b23427942f4a7dede27626d9e1b, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/e450e0ca67304e95b7df979cf7ad9a7b] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp, totalSize=35.9 K 2024-11-14T06:18:43,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on d062eac452eae155317450043cfef448 2024-11-14T06:18:43,283 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d062eac452eae155317450043cfef448 3/3 column families, dataSize=80.70 KB heapSize=224.70 KB 2024-11-14T06:18:43,283 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93657 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:18:43,283 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): d062eac452eae155317450043cfef448/A is initiating minor compaction (all files) 2024-11-14T06:18:43,283 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d062eac452eae155317450043cfef448/A in TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:43,283 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/aa0321ef1598490e989b079f14c4c0b7, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/cf7c6618f20e45969f905c013f13850d, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/8e7acf0dc14047c78dd1894ef528e31b] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp, totalSize=91.5 K 2024-11-14T06:18:43,283 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:43,283 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. files: [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/aa0321ef1598490e989b079f14c4c0b7, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/cf7c6618f20e45969f905c013f13850d, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/8e7acf0dc14047c78dd1894ef528e31b] 2024-11-14T06:18:43,283 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 67d4a97def144407971d9e69d5302ed5, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731565120136 2024-11-14T06:18:43,284 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa0321ef1598490e989b079f14c4c0b7, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731565120136 2024-11-14T06:18:43,284 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 06ba1b23427942f4a7dede27626d9e1b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731565120800 2024-11-14T06:18:43,284 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf7c6618f20e45969f905c013f13850d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731565120800 2024-11-14T06:18:43,284 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting e450e0ca67304e95b7df979cf7ad9a7b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731565121566 2024-11-14T06:18:43,284 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e7acf0dc14047c78dd1894ef528e31b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731565121566 2024-11-14T06:18:43,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=A 2024-11-14T06:18:43,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:18:43,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=B 2024-11-14T06:18:43,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:18:43,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=C 2024-11-14T06:18:43,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T06:18:43,288 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-14T06:18:43,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:43,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. as already flushing 2024-11-14T06:18:43,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:43,289 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:43,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:43,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:43,292 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d062eac452eae155317450043cfef448#B#compaction#513 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:18:43,292 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/c459366c1ccc42f8921d54e41346480a is 50, key is test_row_0/B:col10/1731565121566/Put/seqid=0 2024-11-14T06:18:43,295 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d062eac452eae155317450043cfef448] 2024-11-14T06:18:43,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114c358b3fd1b5e42608cc2e99e00ecdace_d062eac452eae155317450043cfef448 is 50, key is test_row_0/A:col10/1731565123270/Put/seqid=0 2024-11-14T06:18:43,301 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411142f32d1c7a77f49d4a454f541d919f387_d062eac452eae155317450043cfef448 store=[table=TestAcidGuarantees family=A region=d062eac452eae155317450043cfef448] 2024-11-14T06:18:43,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742429_1605 (size=12595) 2024-11-14T06:18:43,303 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411142f32d1c7a77f49d4a454f541d919f387_d062eac452eae155317450043cfef448, store=[table=TestAcidGuarantees family=A region=d062eac452eae155317450043cfef448] 2024-11-14T06:18:43,303 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411142f32d1c7a77f49d4a454f541d919f387_d062eac452eae155317450043cfef448 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d062eac452eae155317450043cfef448] 2024-11-14T06:18:43,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742430_1606 (size=12454) 2024-11-14T06:18:43,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742431_1607 (size=4469) 2024-11-14T06:18:43,309 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:18:43,310 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/c459366c1ccc42f8921d54e41346480a as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/c459366c1ccc42f8921d54e41346480a 2024-11-14T06:18:43,312 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114c358b3fd1b5e42608cc2e99e00ecdace_d062eac452eae155317450043cfef448 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c358b3fd1b5e42608cc2e99e00ecdace_d062eac452eae155317450043cfef448 2024-11-14T06:18:43,313 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/5bc89321357544138a130c60eede428c, store: [table=TestAcidGuarantees family=A region=d062eac452eae155317450043cfef448] 2024-11-14T06:18:43,314 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/5bc89321357544138a130c60eede428c is 175, key is test_row_0/A:col10/1731565123270/Put/seqid=0 2024-11-14T06:18:43,315 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d062eac452eae155317450043cfef448/B of d062eac452eae155317450043cfef448 into c459366c1ccc42f8921d54e41346480a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:18:43,315 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d062eac452eae155317450043cfef448: 2024-11-14T06:18:43,315 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., storeName=d062eac452eae155317450043cfef448/B, priority=13, startTime=1731565123282; duration=0sec 2024-11-14T06:18:43,315 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:18:43,315 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d062eac452eae155317450043cfef448:B 2024-11-14T06:18:43,315 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-14T06:18:43,315 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-14T06:18:43,315 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-14T06:18:43,315 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. because compaction request was cancelled 2024-11-14T06:18:43,315 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d062eac452eae155317450043cfef448:C 2024-11-14T06:18:43,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:43,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565183306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:43,318 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:43,318 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:43,318 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:43,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:43,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 258 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565183310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:43,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:43,320 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:43,320 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:43,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565183313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:43,320 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:43,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:43,320 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:43,320 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:43,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565183314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:43,320 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:43,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:43,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:43,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:43,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:43,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52978 deadline: 1731565183316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:43,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:43,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:43,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:43,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742432_1608 (size=31255) 2024-11-14T06:18:43,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:43,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 259 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565183428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:43,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:43,430 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:43,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565183428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:43,430 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:43,430 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:43,431 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:43,431 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:43,431 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:43,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:43,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 250 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565183428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:43,431 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:43,431 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:43,431 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:43,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:43,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565183428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:43,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:43,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52978 deadline: 1731565183428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:43,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:43,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:43,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:43,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:43,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:43,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:43,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-14T06:18:43,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:43,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. as already flushing 2024-11-14T06:18:43,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:43,442 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:43,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:43,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:43,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-14T06:18:43,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:43,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. as already flushing 2024-11-14T06:18:43,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:43,596 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:43,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:43,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:43,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:43,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 260 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565183637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:43,640 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:43,640 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:43,640 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:43,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:43,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565183638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:43,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:43,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:43,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 251 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565183638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:43,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:43,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:43,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:43,642 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:43,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565183638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:43,642 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:43,642 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:43,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:43,642 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:43,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52978 deadline: 1731565183638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:43,642 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:43,642 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:43,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:43,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:43,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:43,710 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d062eac452eae155317450043cfef448#A#compaction#514 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:18:43,710 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/143491bc728d460290cd1d1dc97b028e is 175, key is test_row_0/A:col10/1731565121566/Put/seqid=0 2024-11-14T06:18:43,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742433_1609 (size=31549) 2024-11-14T06:18:43,726 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=1041, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/5bc89321357544138a130c60eede428c 2024-11-14T06:18:43,733 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/44c6f599aa044d87a7f7559fc4aa0765 is 50, key is test_row_0/B:col10/1731565123270/Put/seqid=0 2024-11-14T06:18:43,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742434_1610 (size=12301) 2024-11-14T06:18:43,748 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-14T06:18:43,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:43,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. as already flushing 2024-11-14T06:18:43,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:43,749 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:43,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:43,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:43,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-14T06:18:43,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:43,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. as already flushing 2024-11-14T06:18:43,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:43,907 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:43,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:43,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:43,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:43,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 261 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565183947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:43,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:43,952 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:43,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565183947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:43,952 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:43,952 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:43,952 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:43,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:43,952 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:43,952 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:43,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565183947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:43,952 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:43,952 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:43,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:43,952 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:43,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565183947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:43,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:43,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52978 deadline: 1731565183948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:43,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:43,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:43,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:43,960 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:43,960 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:43,960 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:44,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-14T06:18:44,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:44,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. as already flushing 2024-11-14T06:18:44,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:44,061 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:44,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:44,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:44,118 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/143491bc728d460290cd1d1dc97b028e as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/143491bc728d460290cd1d1dc97b028e 2024-11-14T06:18:44,121 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d062eac452eae155317450043cfef448/A of d062eac452eae155317450043cfef448 into 143491bc728d460290cd1d1dc97b028e(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:18:44,121 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d062eac452eae155317450043cfef448: 2024-11-14T06:18:44,121 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., storeName=d062eac452eae155317450043cfef448/A, priority=13, startTime=1731565123281; duration=0sec 2024-11-14T06:18:44,121 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:44,121 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d062eac452eae155317450043cfef448:A 2024-11-14T06:18:44,138 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.05 KB at sequenceid=1041 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/44c6f599aa044d87a7f7559fc4aa0765 2024-11-14T06:18:44,149 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/db9b9f15dd2043bfae34c215315c24f8 is 50, key is test_row_0/C:col10/1731565123270/Put/seqid=0 2024-11-14T06:18:44,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742435_1611 (size=12301) 2024-11-14T06:18:44,164 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.05 KB at sequenceid=1041 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/db9b9f15dd2043bfae34c215315c24f8 2024-11-14T06:18:44,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/5bc89321357544138a130c60eede428c as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/5bc89321357544138a130c60eede428c 2024-11-14T06:18:44,173 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/5bc89321357544138a130c60eede428c, entries=150, sequenceid=1041, filesize=30.5 K 2024-11-14T06:18:44,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/44c6f599aa044d87a7f7559fc4aa0765 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/44c6f599aa044d87a7f7559fc4aa0765 2024-11-14T06:18:44,176 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/44c6f599aa044d87a7f7559fc4aa0765, entries=150, sequenceid=1041, filesize=12.0 K 2024-11-14T06:18:44,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/db9b9f15dd2043bfae34c215315c24f8 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/db9b9f15dd2043bfae34c215315c24f8 2024-11-14T06:18:44,181 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/db9b9f15dd2043bfae34c215315c24f8, entries=150, sequenceid=1041, filesize=12.0 K 2024-11-14T06:18:44,181 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.41 KB/89504, heapSize ~244.58 KB/250448, currentSize=93.93 KB/96180 for d062eac452eae155317450043cfef448 in 898ms, sequenceid=1041, compaction requested=true 2024-11-14T06:18:44,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d062eac452eae155317450043cfef448: 2024-11-14T06:18:44,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d062eac452eae155317450043cfef448:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T06:18:44,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:44,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d062eac452eae155317450043cfef448:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T06:18:44,181 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-14T06:18:44,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:44,181 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-14T06:18:44,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d062eac452eae155317450043cfef448:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T06:18:44,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:18:44,182 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-14T06:18:44,182 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-14T06:18:44,182 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-14T06:18:44,182 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-14T06:18:44,182 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. because compaction request was cancelled 2024-11-14T06:18:44,182 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. because compaction request was cancelled 2024-11-14T06:18:44,182 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d062eac452eae155317450043cfef448:B 2024-11-14T06:18:44,182 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d062eac452eae155317450043cfef448:A 2024-11-14T06:18:44,182 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:18:44,183 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36979 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:18:44,183 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): d062eac452eae155317450043cfef448/C is initiating minor compaction (all files) 2024-11-14T06:18:44,183 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d062eac452eae155317450043cfef448/C in TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:44,183 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/7cf09b05e01743eb92024d5f03c8faeb, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/00991d7cbaa14366b6b5bd73102565ba, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/db9b9f15dd2043bfae34c215315c24f8] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp, totalSize=36.1 K 2024-11-14T06:18:44,183 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 7cf09b05e01743eb92024d5f03c8faeb, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731565120800 2024-11-14T06:18:44,183 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 00991d7cbaa14366b6b5bd73102565ba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731565121566 2024-11-14T06:18:44,183 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting db9b9f15dd2043bfae34c215315c24f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1041, earliestPutTs=1731565123270 2024-11-14T06:18:44,194 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d062eac452eae155317450043cfef448#C#compaction#518 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:18:44,195 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/2be3ddd958854833b55ae6d59f42130b is 50, key is test_row_0/C:col10/1731565123270/Put/seqid=0 2024-11-14T06:18:44,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742436_1612 (size=12779) 2024-11-14T06:18:44,214 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-14T06:18:44,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:44,214 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing d062eac452eae155317450043cfef448 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-14T06:18:44,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=A 2024-11-14T06:18:44,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:44,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=B 2024-11-14T06:18:44,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:44,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=C 2024-11-14T06:18:44,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:44,220 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/2be3ddd958854833b55ae6d59f42130b as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/2be3ddd958854833b55ae6d59f42130b 2024-11-14T06:18:44,227 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d062eac452eae155317450043cfef448/C of d062eac452eae155317450043cfef448 into 2be3ddd958854833b55ae6d59f42130b(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:18:44,227 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d062eac452eae155317450043cfef448: 2024-11-14T06:18:44,227 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., storeName=d062eac452eae155317450043cfef448/C, priority=13, startTime=1731565124181; duration=0sec 2024-11-14T06:18:44,227 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:44,227 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d062eac452eae155317450043cfef448:C 2024-11-14T06:18:44,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114f22dd01b0d374ddca8b1219678cc2d0b_d062eac452eae155317450043cfef448 is 50, key is test_row_0/A:col10/1731565123309/Put/seqid=0 2024-11-14T06:18:44,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742437_1613 (size=12454) 2024-11-14T06:18:44,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. as already flushing 2024-11-14T06:18:44,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on d062eac452eae155317450043cfef448 2024-11-14T06:18:44,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:44,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565184499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:44,505 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:44,505 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:44,505 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:44,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:44,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52978 deadline: 1731565184504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:44,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:44,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:44,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:44,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565184505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:44,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:44,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:44,511 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:44,511 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:44,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 266 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565184505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:44,511 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:44,511 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:44,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:44,511 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:44,512 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:44,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565184506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:44,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:44,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:44,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:44,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:44,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 256 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565184607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:44,610 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:44,610 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:44,610 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:44,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:44,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52978 deadline: 1731565184620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:44,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:44,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:44,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 267 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565184621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:44,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:44,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:44,622 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:44,622 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:44,622 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:44,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:44,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565184621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:44,623 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:44,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:44,623 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:44,623 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:44,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565184621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:44,623 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:44,623 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:44,623 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:44,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:18:44,669 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114f22dd01b0d374ddca8b1219678cc2d0b_d062eac452eae155317450043cfef448 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f22dd01b0d374ddca8b1219678cc2d0b_d062eac452eae155317450043cfef448 2024-11-14T06:18:44,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/ceb397c17c7a49e3963f0c9372a84187, store: [table=TestAcidGuarantees family=A region=d062eac452eae155317450043cfef448] 2024-11-14T06:18:44,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/ceb397c17c7a49e3963f0c9372a84187 is 175, key is test_row_0/A:col10/1731565123309/Put/seqid=0 2024-11-14T06:18:44,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742438_1614 (size=31255) 2024-11-14T06:18:44,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:44,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 257 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565184817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:44,822 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:44,822 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:44,822 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:44,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:44,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 268 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565184827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:44,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:44,831 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:44,831 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:44,831 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:44,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52978 deadline: 1731565184827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:44,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:44,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:44,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:44,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:44,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565184827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:44,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:44,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:44,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:44,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:44,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565184827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:44,832 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:44,832 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:44,832 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:44,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-14T06:18:45,102 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=1060, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/ceb397c17c7a49e3963f0c9372a84187 2024-11-14T06:18:45,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/d1afe333131d46f1bffc4ba3931751c5 is 50, key is test_row_0/B:col10/1731565123309/Put/seqid=0 2024-11-14T06:18:45,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742439_1615 (size=12301) 2024-11-14T06:18:45,112 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=1060 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/d1afe333131d46f1bffc4ba3931751c5 2024-11-14T06:18:45,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/e0011cb0fa4b4ad4b50d0935fac8f92a is 50, key is test_row_0/C:col10/1731565123309/Put/seqid=0 2024-11-14T06:18:45,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742440_1616 (size=12301) 2024-11-14T06:18:45,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:45,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 258 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565185127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:45,130 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:45,130 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:45,130 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:45,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:45,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 269 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565185137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:45,138 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:45,138 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:45,138 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:45,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:45,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52978 deadline: 1731565185137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:45,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:45,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:45,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:45,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:45,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565185137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:45,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:45,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:45,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:45,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:45,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565185137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:45,141 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:45,141 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:45,141 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:45,528 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=1060 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/e0011cb0fa4b4ad4b50d0935fac8f92a 2024-11-14T06:18:45,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/ceb397c17c7a49e3963f0c9372a84187 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/ceb397c17c7a49e3963f0c9372a84187 2024-11-14T06:18:45,538 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/ceb397c17c7a49e3963f0c9372a84187, entries=150, sequenceid=1060, filesize=30.5 K 2024-11-14T06:18:45,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/d1afe333131d46f1bffc4ba3931751c5 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/d1afe333131d46f1bffc4ba3931751c5 2024-11-14T06:18:45,542 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/d1afe333131d46f1bffc4ba3931751c5, entries=150, sequenceid=1060, filesize=12.0 K 2024-11-14T06:18:45,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/e0011cb0fa4b4ad4b50d0935fac8f92a as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/e0011cb0fa4b4ad4b50d0935fac8f92a 2024-11-14T06:18:45,547 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/e0011cb0fa4b4ad4b50d0935fac8f92a, entries=150, sequenceid=1060, filesize=12.0 K 2024-11-14T06:18:45,548 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=93.93 KB/96180 for d062eac452eae155317450043cfef448 in 1334ms, sequenceid=1060, compaction requested=true 2024-11-14T06:18:45,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for d062eac452eae155317450043cfef448: 2024-11-14T06:18:45,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:45,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-14T06:18:45,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-14T06:18:45,552 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-14T06:18:45,552 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.7190 sec 2024-11-14T06:18:45,553 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 4.7240 sec 2024-11-14T06:18:45,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on d062eac452eae155317450043cfef448 2024-11-14T06:18:45,641 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d062eac452eae155317450043cfef448 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-14T06:18:45,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=A 2024-11-14T06:18:45,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:45,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=B 2024-11-14T06:18:45,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:45,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=C 2024-11-14T06:18:45,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:45,663 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114face7b18fe374530b71942f689afe366_d062eac452eae155317450043cfef448 is 50, key is test_row_0/A:col10/1731565124503/Put/seqid=0 2024-11-14T06:18:45,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:45,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565185658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:45,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:45,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:45,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:45,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:45,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 271 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565185660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:45,667 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:45,667 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:45,667 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:45,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:45,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565185664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:45,671 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:45,671 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:45,671 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:45,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:45,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 264 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565185665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:45,673 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:45,673 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:45,673 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:45,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742441_1617 (size=14994) 2024-11-14T06:18:45,674 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:18:45,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:45,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52978 deadline: 1731565185667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:45,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:45,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:45,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:45,678 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114face7b18fe374530b71942f689afe366_d062eac452eae155317450043cfef448 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114face7b18fe374530b71942f689afe366_d062eac452eae155317450043cfef448 2024-11-14T06:18:45,679 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/b58735e053ee4b86a217e25250c7e1e2, store: [table=TestAcidGuarantees family=A region=d062eac452eae155317450043cfef448] 2024-11-14T06:18:45,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/b58735e053ee4b86a217e25250c7e1e2 is 175, key is test_row_0/A:col10/1731565124503/Put/seqid=0 2024-11-14T06:18:45,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742442_1618 (size=39949) 2024-11-14T06:18:45,699 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=1079, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/b58735e053ee4b86a217e25250c7e1e2 2024-11-14T06:18:45,722 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/d1a3ecb5318841d1888bdf3c0a2853ba is 50, key is test_row_0/B:col10/1731565124503/Put/seqid=0 2024-11-14T06:18:45,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742443_1619 (size=12301) 2024-11-14T06:18:45,763 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=1079 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/d1a3ecb5318841d1888bdf3c0a2853ba 2024-11-14T06:18:45,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:45,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 265 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565185777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:45,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:45,784 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:45,785 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:45,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565185777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:45,785 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:45,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:45,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:45,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:45,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:45,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:45,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565185778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:45,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 272 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565185778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:45,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:45,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52978 deadline: 1731565185778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:45,785 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:45,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:45,785 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:45,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:45,786 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:45,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:45,786 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:45,786 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:45,786 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:45,792 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/c61547414e684c27b4c65366a7c18fe7 is 50, key is test_row_0/C:col10/1731565124503/Put/seqid=0 2024-11-14T06:18:45,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742444_1620 (size=12301) 2024-11-14T06:18:45,826 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=1079 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/c61547414e684c27b4c65366a7c18fe7 2024-11-14T06:18:45,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/b58735e053ee4b86a217e25250c7e1e2 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/b58735e053ee4b86a217e25250c7e1e2 2024-11-14T06:18:45,834 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/b58735e053ee4b86a217e25250c7e1e2, entries=200, sequenceid=1079, filesize=39.0 K 2024-11-14T06:18:45,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/d1a3ecb5318841d1888bdf3c0a2853ba as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/d1a3ecb5318841d1888bdf3c0a2853ba 2024-11-14T06:18:45,839 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/d1a3ecb5318841d1888bdf3c0a2853ba, entries=150, sequenceid=1079, filesize=12.0 K 2024-11-14T06:18:45,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/c61547414e684c27b4c65366a7c18fe7 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/c61547414e684c27b4c65366a7c18fe7 2024-11-14T06:18:45,849 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/c61547414e684c27b4c65366a7c18fe7, entries=150, sequenceid=1079, filesize=12.0 K 2024-11-14T06:18:45,851 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for d062eac452eae155317450043cfef448 in 210ms, sequenceid=1079, compaction requested=true 2024-11-14T06:18:45,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d062eac452eae155317450043cfef448: 2024-11-14T06:18:45,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d062eac452eae155317450043cfef448:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T06:18:45,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:45,851 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T06:18:45,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d062eac452eae155317450043cfef448:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T06:18:45,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:45,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d062eac452eae155317450043cfef448:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T06:18:45,851 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T06:18:45,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:18:45,852 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134008 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T06:18:45,852 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): d062eac452eae155317450043cfef448/A is initiating minor compaction (all files) 2024-11-14T06:18:45,852 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49498 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T06:18:45,852 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d062eac452eae155317450043cfef448/A in TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:45,852 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): d062eac452eae155317450043cfef448/B is initiating minor compaction (all files) 2024-11-14T06:18:45,852 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d062eac452eae155317450043cfef448/B in TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:45,852 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/143491bc728d460290cd1d1dc97b028e, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/5bc89321357544138a130c60eede428c, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/ceb397c17c7a49e3963f0c9372a84187, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/b58735e053ee4b86a217e25250c7e1e2] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp, totalSize=130.9 K 2024-11-14T06:18:45,852 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/c459366c1ccc42f8921d54e41346480a, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/44c6f599aa044d87a7f7559fc4aa0765, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/d1afe333131d46f1bffc4ba3931751c5, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/d1a3ecb5318841d1888bdf3c0a2853ba] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp, totalSize=48.3 K 2024-11-14T06:18:45,852 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:45,852 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. files: [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/143491bc728d460290cd1d1dc97b028e, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/5bc89321357544138a130c60eede428c, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/ceb397c17c7a49e3963f0c9372a84187, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/b58735e053ee4b86a217e25250c7e1e2] 2024-11-14T06:18:45,852 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting c459366c1ccc42f8921d54e41346480a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731565121566 2024-11-14T06:18:45,852 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 143491bc728d460290cd1d1dc97b028e, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731565121566 2024-11-14T06:18:45,853 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 44c6f599aa044d87a7f7559fc4aa0765, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1041, earliestPutTs=1731565123270 2024-11-14T06:18:45,853 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5bc89321357544138a130c60eede428c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=1041, earliestPutTs=1731565123270 2024-11-14T06:18:45,856 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting d1afe333131d46f1bffc4ba3931751c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1060, earliestPutTs=1731565123295 2024-11-14T06:18:45,856 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting ceb397c17c7a49e3963f0c9372a84187, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=1060, earliestPutTs=1731565123295 2024-11-14T06:18:45,856 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting d1a3ecb5318841d1888bdf3c0a2853ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1079, earliestPutTs=1731565124495 2024-11-14T06:18:45,856 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting b58735e053ee4b86a217e25250c7e1e2, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=1079, earliestPutTs=1731565124495 2024-11-14T06:18:45,863 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d062eac452eae155317450043cfef448#B#compaction#525 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:18:45,864 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/9c4070c8c814480db653fa73ccc356da is 50, key is test_row_0/B:col10/1731565124503/Put/seqid=0 2024-11-14T06:18:45,865 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d062eac452eae155317450043cfef448] 2024-11-14T06:18:45,882 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111412d014f189f7416b88bb270010e926dd_d062eac452eae155317450043cfef448 store=[table=TestAcidGuarantees family=A region=d062eac452eae155317450043cfef448] 2024-11-14T06:18:45,884 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111412d014f189f7416b88bb270010e926dd_d062eac452eae155317450043cfef448, store=[table=TestAcidGuarantees family=A region=d062eac452eae155317450043cfef448] 2024-11-14T06:18:45,885 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111412d014f189f7416b88bb270010e926dd_d062eac452eae155317450043cfef448 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d062eac452eae155317450043cfef448] 2024-11-14T06:18:45,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742445_1621 (size=12881) 2024-11-14T06:18:45,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742446_1622 (size=4469) 2024-11-14T06:18:45,917 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/9c4070c8c814480db653fa73ccc356da as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/9c4070c8c814480db653fa73ccc356da 2024-11-14T06:18:45,924 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d062eac452eae155317450043cfef448/B of d062eac452eae155317450043cfef448 into 9c4070c8c814480db653fa73ccc356da(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:18:45,924 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d062eac452eae155317450043cfef448: 2024-11-14T06:18:45,924 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., storeName=d062eac452eae155317450043cfef448/B, priority=12, startTime=1731565125851; duration=0sec 2024-11-14T06:18:45,924 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:18:45,925 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d062eac452eae155317450043cfef448:B 2024-11-14T06:18:45,925 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:18:45,926 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:18:45,926 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): d062eac452eae155317450043cfef448/C is initiating minor compaction (all files) 2024-11-14T06:18:45,926 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d062eac452eae155317450043cfef448/C in TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:45,926 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/2be3ddd958854833b55ae6d59f42130b, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/e0011cb0fa4b4ad4b50d0935fac8f92a, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/c61547414e684c27b4c65366a7c18fe7] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp, totalSize=36.5 K 2024-11-14T06:18:45,927 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 2be3ddd958854833b55ae6d59f42130b, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=1041, earliestPutTs=1731565123270 2024-11-14T06:18:45,927 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting e0011cb0fa4b4ad4b50d0935fac8f92a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1060, earliestPutTs=1731565123295 2024-11-14T06:18:45,927 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting c61547414e684c27b4c65366a7c18fe7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1079, earliestPutTs=1731565124495 2024-11-14T06:18:45,950 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d062eac452eae155317450043cfef448#C#compaction#527 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:18:45,951 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/f304c828e52c40faa6a949caca08c344 is 50, key is test_row_0/C:col10/1731565124503/Put/seqid=0 2024-11-14T06:18:45,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742447_1623 (size=12881) 2024-11-14T06:18:45,990 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d062eac452eae155317450043cfef448 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-14T06:18:45,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=A 2024-11-14T06:18:45,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:45,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=B 2024-11-14T06:18:45,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:45,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=C 2024-11-14T06:18:45,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:45,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on d062eac452eae155317450043cfef448 2024-11-14T06:18:45,993 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/f304c828e52c40faa6a949caca08c344 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/f304c828e52c40faa6a949caca08c344 2024-11-14T06:18:46,001 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d062eac452eae155317450043cfef448/C of d062eac452eae155317450043cfef448 into f304c828e52c40faa6a949caca08c344(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:18:46,001 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d062eac452eae155317450043cfef448: 2024-11-14T06:18:46,001 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., storeName=d062eac452eae155317450043cfef448/C, priority=13, startTime=1731565125851; duration=0sec 2024-11-14T06:18:46,001 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:46,001 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d062eac452eae155317450043cfef448:C 2024-11-14T06:18:46,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411142b160948ea044e13acde8c2372ade99a_d062eac452eae155317450043cfef448 is 50, key is test_row_0/A:col10/1731565125662/Put/seqid=0 2024-11-14T06:18:46,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:46,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 270 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565186016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:46,025 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:46,025 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:46,025 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:46,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:46,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565186017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:46,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742448_1624 (size=14994) 2024-11-14T06:18:46,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:46,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:46,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:46,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:46,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 275 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565186017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:46,027 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:46,027 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:46,027 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:46,029 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:18:46,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:46,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52978 deadline: 1731565186025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:46,032 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411142b160948ea044e13acde8c2372ade99a_d062eac452eae155317450043cfef448 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411142b160948ea044e13acde8c2372ade99a_d062eac452eae155317450043cfef448 2024-11-14T06:18:46,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:46,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:46,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:46,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:46,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565186025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:46,033 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:46,033 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:46,033 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:46,033 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/848d7e4ec4574d5db96e77ef12145ff4, store: [table=TestAcidGuarantees family=A region=d062eac452eae155317450043cfef448] 2024-11-14T06:18:46,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/848d7e4ec4574d5db96e77ef12145ff4 is 175, key is test_row_0/A:col10/1731565125662/Put/seqid=0 2024-11-14T06:18:46,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742449_1625 (size=39949) 2024-11-14T06:18:46,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:46,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 271 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565186137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:46,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:46,139 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:46,139 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:46,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565186137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:46,139 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:46,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:46,140 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:46,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565186137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:46,140 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:46,140 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:46,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:46,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:46,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:46,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:46,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 276 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565186137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:46,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:46,140 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:46,140 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:46,140 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:46,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52978 deadline: 1731565186137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:46,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:46,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:46,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:46,307 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d062eac452eae155317450043cfef448#A#compaction#526 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:18:46,308 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/47d62f1beabf46f9a5114842d263684e is 175, key is test_row_0/A:col10/1731565124503/Put/seqid=0 2024-11-14T06:18:46,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742450_1626 (size=31835) 2024-11-14T06:18:46,318 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/47d62f1beabf46f9a5114842d263684e as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/47d62f1beabf46f9a5114842d263684e 2024-11-14T06:18:46,323 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d062eac452eae155317450043cfef448/A of d062eac452eae155317450043cfef448 into 47d62f1beabf46f9a5114842d263684e(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:18:46,323 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d062eac452eae155317450043cfef448: 2024-11-14T06:18:46,323 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., storeName=d062eac452eae155317450043cfef448/A, priority=12, startTime=1731565125851; duration=0sec 2024-11-14T06:18:46,323 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:46,323 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d062eac452eae155317450043cfef448:A 2024-11-14T06:18:46,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:46,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 272 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565186347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:46,349 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:46,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:46,349 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:46,349 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:46,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565186347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:46,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:46,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:46,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:46,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:46,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52978 deadline: 1731565186347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:46,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:46,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:46,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:46,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565186348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:46,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:46,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:46,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 277 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565186348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:46,351 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:46,351 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:46,351 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:46,352 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:46,352 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:46,352 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:46,459 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=1096, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/848d7e4ec4574d5db96e77ef12145ff4 2024-11-14T06:18:46,466 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/9f0bd28752614a0e8247e28e77180b7e is 50, key is test_row_0/B:col10/1731565125662/Put/seqid=0 2024-11-14T06:18:46,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742451_1627 (size=12301) 2024-11-14T06:18:46,469 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=1096 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/9f0bd28752614a0e8247e28e77180b7e 2024-11-14T06:18:46,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/36efa621b213439c9bbd76d6627c1f17 is 50, key is test_row_0/C:col10/1731565125662/Put/seqid=0 2024-11-14T06:18:46,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742452_1628 (size=12301) 2024-11-14T06:18:46,478 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=1096 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/36efa621b213439c9bbd76d6627c1f17 2024-11-14T06:18:46,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/848d7e4ec4574d5db96e77ef12145ff4 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/848d7e4ec4574d5db96e77ef12145ff4 2024-11-14T06:18:46,484 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/848d7e4ec4574d5db96e77ef12145ff4, entries=200, sequenceid=1096, filesize=39.0 K 2024-11-14T06:18:46,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/9f0bd28752614a0e8247e28e77180b7e as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/9f0bd28752614a0e8247e28e77180b7e 2024-11-14T06:18:46,488 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/9f0bd28752614a0e8247e28e77180b7e, entries=150, sequenceid=1096, filesize=12.0 K 2024-11-14T06:18:46,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/36efa621b213439c9bbd76d6627c1f17 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/36efa621b213439c9bbd76d6627c1f17 2024-11-14T06:18:46,491 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/36efa621b213439c9bbd76d6627c1f17, entries=150, sequenceid=1096, filesize=12.0 K 2024-11-14T06:18:46,492 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for d062eac452eae155317450043cfef448 in 502ms, sequenceid=1096, compaction requested=false 2024-11-14T06:18:46,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d062eac452eae155317450043cfef448: 2024-11-14T06:18:46,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on d062eac452eae155317450043cfef448 2024-11-14T06:18:46,659 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d062eac452eae155317450043cfef448 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-14T06:18:46,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=A 2024-11-14T06:18:46,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:46,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=B 2024-11-14T06:18:46,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:46,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=C 2024-11-14T06:18:46,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:46,668 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411140fe235d14d4c48c8bf58488625b45796_d062eac452eae155317450043cfef448 is 50, key is test_row_0/A:col10/1731565126015/Put/seqid=0 2024-11-14T06:18:46,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:46,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 257 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565186672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:46,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:46,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52978 deadline: 1731565186674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:46,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:46,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:46,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:46,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 276 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565186675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:46,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:46,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:46,680 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:46,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:46,680 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:46,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:46,680 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:46,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:46,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 281 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565186679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:46,686 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:46,686 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:46,686 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:46,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:46,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565186681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:46,687 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:46,687 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:46,687 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:46,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742453_1629 (size=14994) 2024-11-14T06:18:46,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:46,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52978 deadline: 1731565186788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:46,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:46,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:46,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 258 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565186788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:46,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:46,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:46,792 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:46,792 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:46,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:46,792 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:46,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 277 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565186788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:46,792 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:46,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:46,792 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:46,792 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:46,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 282 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565186788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:46,793 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:46,793 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:46,793 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:46,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:46,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565186797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:46,805 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:46,805 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:46,805 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:47,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:47,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52978 deadline: 1731565186997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:47,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:47,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:47,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:47,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:47,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 283 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565186998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:47,001 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:47,002 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:47,002 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:47,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:47,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 259 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565186998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:47,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:47,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:47,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:47,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:47,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 278 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565186998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:47,002 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:47,002 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:47,002 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:47,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:47,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565187017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:47,019 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:47,019 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:47,019 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:47,111 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:18:47,118 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411140fe235d14d4c48c8bf58488625b45796_d062eac452eae155317450043cfef448 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411140fe235d14d4c48c8bf58488625b45796_d062eac452eae155317450043cfef448 2024-11-14T06:18:47,119 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/82936432b5474561973b079d21605b91, store: [table=TestAcidGuarantees family=A region=d062eac452eae155317450043cfef448] 2024-11-14T06:18:47,120 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/82936432b5474561973b079d21605b91 is 175, key is test_row_0/A:col10/1731565126015/Put/seqid=0 2024-11-14T06:18:47,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742454_1630 (size=39949) 2024-11-14T06:18:47,131 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=1115, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/82936432b5474561973b079d21605b91 2024-11-14T06:18:47,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/d9f574421bc64fde81f93c056385a51c is 50, key is test_row_0/B:col10/1731565126015/Put/seqid=0 2024-11-14T06:18:47,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742455_1631 (size=12301) 2024-11-14T06:18:47,158 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=1115 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/d9f574421bc64fde81f93c056385a51c 2024-11-14T06:18:47,166 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/1fd2d8f9c0a149b29db5148a0d181b57 is 50, key is test_row_0/C:col10/1731565126015/Put/seqid=0 2024-11-14T06:18:47,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742456_1632 (size=12301) 2024-11-14T06:18:47,190 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=1115 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/1fd2d8f9c0a149b29db5148a0d181b57 2024-11-14T06:18:47,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/82936432b5474561973b079d21605b91 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/82936432b5474561973b079d21605b91 2024-11-14T06:18:47,198 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/82936432b5474561973b079d21605b91, entries=200, sequenceid=1115, filesize=39.0 K 2024-11-14T06:18:47,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/d9f574421bc64fde81f93c056385a51c as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/d9f574421bc64fde81f93c056385a51c 2024-11-14T06:18:47,203 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/d9f574421bc64fde81f93c056385a51c, entries=150, sequenceid=1115, filesize=12.0 K 2024-11-14T06:18:47,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/1fd2d8f9c0a149b29db5148a0d181b57 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/1fd2d8f9c0a149b29db5148a0d181b57 2024-11-14T06:18:47,209 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/1fd2d8f9c0a149b29db5148a0d181b57, entries=150, sequenceid=1115, filesize=12.0 K 2024-11-14T06:18:47,210 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for d062eac452eae155317450043cfef448 in 551ms, sequenceid=1115, compaction requested=true 2024-11-14T06:18:47,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d062eac452eae155317450043cfef448: 2024-11-14T06:18:47,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d062eac452eae155317450043cfef448:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T06:18:47,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:47,210 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:18:47,210 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:18:47,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d062eac452eae155317450043cfef448:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T06:18:47,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:47,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d062eac452eae155317450043cfef448:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T06:18:47,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:18:47,211 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:18:47,211 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): d062eac452eae155317450043cfef448/B is initiating minor compaction (all files) 2024-11-14T06:18:47,211 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d062eac452eae155317450043cfef448/B in TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:47,211 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/9c4070c8c814480db653fa73ccc356da, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/9f0bd28752614a0e8247e28e77180b7e, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/d9f574421bc64fde81f93c056385a51c] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp, totalSize=36.6 K 2024-11-14T06:18:47,212 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111733 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:18:47,212 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): d062eac452eae155317450043cfef448/A is initiating minor compaction (all files) 2024-11-14T06:18:47,212 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d062eac452eae155317450043cfef448/A in TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:47,212 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/47d62f1beabf46f9a5114842d263684e, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/848d7e4ec4574d5db96e77ef12145ff4, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/82936432b5474561973b079d21605b91] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp, totalSize=109.1 K 2024-11-14T06:18:47,212 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:47,212 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. files: [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/47d62f1beabf46f9a5114842d263684e, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/848d7e4ec4574d5db96e77ef12145ff4, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/82936432b5474561973b079d21605b91] 2024-11-14T06:18:47,212 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c4070c8c814480db653fa73ccc356da, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=1079, earliestPutTs=1731565124495 2024-11-14T06:18:47,212 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47d62f1beabf46f9a5114842d263684e, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=1079, earliestPutTs=1731565124495 2024-11-14T06:18:47,213 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f0bd28752614a0e8247e28e77180b7e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1096, earliestPutTs=1731565125648 2024-11-14T06:18:47,213 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 848d7e4ec4574d5db96e77ef12145ff4, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=1096, earliestPutTs=1731565125648 2024-11-14T06:18:47,213 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting d9f574421bc64fde81f93c056385a51c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1115, earliestPutTs=1731565126015 2024-11-14T06:18:47,220 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82936432b5474561973b079d21605b91, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=1115, earliestPutTs=1731565126015 2024-11-14T06:18:47,234 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d062eac452eae155317450043cfef448#B#compaction#534 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:18:47,235 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/6922ec1be880493abf4e32a70992895e is 50, key is test_row_0/B:col10/1731565126015/Put/seqid=0 2024-11-14T06:18:47,237 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d062eac452eae155317450043cfef448] 2024-11-14T06:18:47,253 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411145c6b19713a8841fc80ffb02ba8c8ba69_d062eac452eae155317450043cfef448 store=[table=TestAcidGuarantees family=A region=d062eac452eae155317450043cfef448] 2024-11-14T06:18:47,255 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411145c6b19713a8841fc80ffb02ba8c8ba69_d062eac452eae155317450043cfef448, store=[table=TestAcidGuarantees family=A region=d062eac452eae155317450043cfef448] 2024-11-14T06:18:47,255 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411145c6b19713a8841fc80ffb02ba8c8ba69_d062eac452eae155317450043cfef448 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d062eac452eae155317450043cfef448] 2024-11-14T06:18:47,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742457_1633 (size=12983) 2024-11-14T06:18:47,277 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/6922ec1be880493abf4e32a70992895e as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/6922ec1be880493abf4e32a70992895e 2024-11-14T06:18:47,283 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d062eac452eae155317450043cfef448/B of d062eac452eae155317450043cfef448 into 6922ec1be880493abf4e32a70992895e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:18:47,283 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d062eac452eae155317450043cfef448: 2024-11-14T06:18:47,283 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., storeName=d062eac452eae155317450043cfef448/B, priority=13, startTime=1731565127210; duration=0sec 2024-11-14T06:18:47,283 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:18:47,283 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d062eac452eae155317450043cfef448:B 2024-11-14T06:18:47,283 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:18:47,284 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:18:47,284 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): d062eac452eae155317450043cfef448/C is initiating minor compaction (all files) 2024-11-14T06:18:47,284 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d062eac452eae155317450043cfef448/C in TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:47,284 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/f304c828e52c40faa6a949caca08c344, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/36efa621b213439c9bbd76d6627c1f17, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/1fd2d8f9c0a149b29db5148a0d181b57] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp, totalSize=36.6 K 2024-11-14T06:18:47,284 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting f304c828e52c40faa6a949caca08c344, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=1079, earliestPutTs=1731565124495 2024-11-14T06:18:47,285 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 36efa621b213439c9bbd76d6627c1f17, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1096, earliestPutTs=1731565125648 2024-11-14T06:18:47,285 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fd2d8f9c0a149b29db5148a0d181b57, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1115, earliestPutTs=1731565126015 2024-11-14T06:18:47,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742458_1634 (size=4469) 2024-11-14T06:18:47,313 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d062eac452eae155317450043cfef448 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-14T06:18:47,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on d062eac452eae155317450043cfef448 2024-11-14T06:18:47,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=A 2024-11-14T06:18:47,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:47,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=B 2024-11-14T06:18:47,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:47,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=C 2024-11-14T06:18:47,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:47,322 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d062eac452eae155317450043cfef448#C#compaction#536 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:18:47,323 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/d52c18fbf82141c08b05496681bc0b89 is 50, key is test_row_0/C:col10/1731565126015/Put/seqid=0 2024-11-14T06:18:47,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:47,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:47,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52978 deadline: 1731565187331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:47,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 263 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565187331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:47,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:47,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 282 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565187337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:47,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:47,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:47,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:47,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:47,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 287 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565187338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:47,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:47,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:47,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:47,344 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:47,344 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:47,344 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:47,344 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:47,344 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:47,344 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:47,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:47,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 257 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565187341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:47,346 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:47,346 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:47,346 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:47,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742459_1635 (size=12983) 2024-11-14T06:18:47,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114d64b3fcd48934ff8a324d48fd50a97df_d062eac452eae155317450043cfef448 is 50, key is test_row_0/A:col10/1731565127312/Put/seqid=0 2024-11-14T06:18:47,367 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/d52c18fbf82141c08b05496681bc0b89 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/d52c18fbf82141c08b05496681bc0b89 2024-11-14T06:18:47,372 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d062eac452eae155317450043cfef448/C of d062eac452eae155317450043cfef448 into d52c18fbf82141c08b05496681bc0b89(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:18:47,373 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d062eac452eae155317450043cfef448: 2024-11-14T06:18:47,373 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., storeName=d062eac452eae155317450043cfef448/C, priority=13, startTime=1731565127211; duration=0sec 2024-11-14T06:18:47,373 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:47,373 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d062eac452eae155317450043cfef448:C 2024-11-14T06:18:47,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742460_1636 (size=12454) 2024-11-14T06:18:47,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:47,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 283 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565187448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:47,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:47,450 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:47,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52978 deadline: 1731565187448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:47,451 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:47,451 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:47,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:47,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:47,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 264 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565187448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:47,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:47,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:47,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:47,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:47,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:47,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:47,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 288 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565187448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:47,451 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:47,451 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:47,451 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:47,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:47,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 258 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565187457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:47,458 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:47,458 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:47,458 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:47,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:47,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 265 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565187657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:47,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:47,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:47,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:47,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 284 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565187658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:47,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:47,662 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:47,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:47,662 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:47,662 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:47,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52978 deadline: 1731565187658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:47,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:47,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:47,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:47,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:47,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 289 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565187658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:47,662 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:47,662 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:47,662 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:47,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:47,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 259 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565187667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:47,668 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:47,668 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:47,668 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:47,706 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d062eac452eae155317450043cfef448#A#compaction#535 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:18:47,707 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/4993d807f91748ca9434a4faad1d349b is 175, key is test_row_0/A:col10/1731565126015/Put/seqid=0 2024-11-14T06:18:47,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742461_1637 (size=31937) 2024-11-14T06:18:47,785 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:18:47,793 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114d64b3fcd48934ff8a324d48fd50a97df_d062eac452eae155317450043cfef448 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d64b3fcd48934ff8a324d48fd50a97df_d062eac452eae155317450043cfef448 2024-11-14T06:18:47,794 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/4f700b3a47ae4817a59b0265c5599483, store: [table=TestAcidGuarantees family=A region=d062eac452eae155317450043cfef448] 2024-11-14T06:18:47,795 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/4f700b3a47ae4817a59b0265c5599483 is 175, key is test_row_0/A:col10/1731565127312/Put/seqid=0 2024-11-14T06:18:47,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742462_1638 (size=31255) 2024-11-14T06:18:47,809 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=1135, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/4f700b3a47ae4817a59b0265c5599483 2024-11-14T06:18:47,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/f45f4d0293f64208b835f3fd01c82e18 is 50, key is test_row_0/B:col10/1731565127312/Put/seqid=0 2024-11-14T06:18:47,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742463_1639 (size=12301) 2024-11-14T06:18:47,843 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=1135 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/f45f4d0293f64208b835f3fd01c82e18 2024-11-14T06:18:47,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/a90a94b66f644432a37f6498f4d86a51 is 50, key is test_row_0/C:col10/1731565127312/Put/seqid=0 2024-11-14T06:18:47,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742464_1640 (size=12301) 2024-11-14T06:18:47,898 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=1135 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/a90a94b66f644432a37f6498f4d86a51 2024-11-14T06:18:47,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/4f700b3a47ae4817a59b0265c5599483 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/4f700b3a47ae4817a59b0265c5599483 2024-11-14T06:18:47,911 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/4f700b3a47ae4817a59b0265c5599483, entries=150, sequenceid=1135, filesize=30.5 K 2024-11-14T06:18:47,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/f45f4d0293f64208b835f3fd01c82e18 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/f45f4d0293f64208b835f3fd01c82e18 2024-11-14T06:18:47,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/f45f4d0293f64208b835f3fd01c82e18, entries=150, sequenceid=1135, filesize=12.0 K 2024-11-14T06:18:47,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/a90a94b66f644432a37f6498f4d86a51 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/a90a94b66f644432a37f6498f4d86a51 2024-11-14T06:18:47,922 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/a90a94b66f644432a37f6498f4d86a51, entries=150, sequenceid=1135, filesize=12.0 K 2024-11-14T06:18:47,922 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for d062eac452eae155317450043cfef448 in 610ms, sequenceid=1135, compaction requested=false 2024-11-14T06:18:47,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d062eac452eae155317450043cfef448: 2024-11-14T06:18:47,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on d062eac452eae155317450043cfef448 2024-11-14T06:18:47,969 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d062eac452eae155317450043cfef448 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-14T06:18:47,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=A 2024-11-14T06:18:47,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:47,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=B 2024-11-14T06:18:47,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:47,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=C 2024-11-14T06:18:47,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:47,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:47,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:47,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 288 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565187987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:47,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52978 deadline: 1731565187989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:47,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:47,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 294 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565187990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:47,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:47,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:47,996 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:47,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:47,996 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:47,996 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:47,996 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:47,996 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:47,996 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:47,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:47,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 262 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565187992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:47,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:47,999 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:47,999 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:47,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 270 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565187993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:47,999 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:47,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:47,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:48,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:48,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411145b8d36e7aa774a979d0e1c641677f689_d062eac452eae155317450043cfef448 is 50, key is test_row_0/A:col10/1731565127334/Put/seqid=0 2024-11-14T06:18:48,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742465_1641 (size=12454) 2024-11-14T06:18:48,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:48,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 289 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565188107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:48,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:48,111 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:48,111 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:48,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52978 deadline: 1731565188108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:48,111 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:48,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:48,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:48,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:48,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:48,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 295 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565188108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:48,112 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:48,112 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:48,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:48,112 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:48,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 271 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565188108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:48,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:48,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:48,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:48,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:48,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 263 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565188108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:48,113 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:48,113 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:48,113 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:48,138 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/4993d807f91748ca9434a4faad1d349b as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/4993d807f91748ca9434a4faad1d349b 2024-11-14T06:18:48,152 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d062eac452eae155317450043cfef448/A of d062eac452eae155317450043cfef448 into 4993d807f91748ca9434a4faad1d349b(size=31.2 K), total size for store is 61.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:18:48,152 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d062eac452eae155317450043cfef448: 2024-11-14T06:18:48,152 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., storeName=d062eac452eae155317450043cfef448/A, priority=13, startTime=1731565127210; duration=0sec 2024-11-14T06:18:48,152 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:48,152 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d062eac452eae155317450043cfef448:A 2024-11-14T06:18:48,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:48,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 290 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565188317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:48,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:48,321 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:48,321 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:48,321 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:48,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 272 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565188318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:48,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:48,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:48,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:48,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:48,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:48,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52978 deadline: 1731565188318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:48,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 296 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565188318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:48,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:48,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:48,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:48,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:48,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 264 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565188318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:48,322 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:48,322 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:48,322 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:48,322 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:48,322 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:48,322 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:48,442 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:18:48,449 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411145b8d36e7aa774a979d0e1c641677f689_d062eac452eae155317450043cfef448 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145b8d36e7aa774a979d0e1c641677f689_d062eac452eae155317450043cfef448 2024-11-14T06:18:48,450 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/774faa2243864c2abeefc2f3a13f1187, store: [table=TestAcidGuarantees family=A region=d062eac452eae155317450043cfef448] 2024-11-14T06:18:48,450 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/774faa2243864c2abeefc2f3a13f1187 is 175, key is test_row_0/A:col10/1731565127334/Put/seqid=0 2024-11-14T06:18:48,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742466_1642 (size=31255) 2024-11-14T06:18:48,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:48,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 297 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565188627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:48,630 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:48,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:48,631 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:48,631 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:48,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 291 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565188627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:48,631 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:48,631 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:48,631 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:48,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:48,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 273 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565188627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:48,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:48,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:48,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:48,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:48,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52978 deadline: 1731565188627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:48,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:48,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:48,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:48,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 265 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565188628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:48,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:48,632 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:48,632 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:48,632 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:48,859 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=1151, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/774faa2243864c2abeefc2f3a13f1187 2024-11-14T06:18:48,865 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/38b38e681147441fbec665fdcea2e121 is 50, key is test_row_0/B:col10/1731565127334/Put/seqid=0 2024-11-14T06:18:48,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742467_1643 (size=12301) 2024-11-14T06:18:48,878 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=1151 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/38b38e681147441fbec665fdcea2e121 2024-11-14T06:18:48,885 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/a2c152b0203e415c8842a73ebccc44fa is 50, key is test_row_0/C:col10/1731565127334/Put/seqid=0 2024-11-14T06:18:48,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742468_1644 (size=12301) 2024-11-14T06:18:48,891 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=1151 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/a2c152b0203e415c8842a73ebccc44fa 2024-11-14T06:18:48,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/774faa2243864c2abeefc2f3a13f1187 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/774faa2243864c2abeefc2f3a13f1187 2024-11-14T06:18:48,899 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/774faa2243864c2abeefc2f3a13f1187, entries=150, sequenceid=1151, filesize=30.5 K 2024-11-14T06:18:48,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/38b38e681147441fbec665fdcea2e121 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/38b38e681147441fbec665fdcea2e121 2024-11-14T06:18:48,915 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/38b38e681147441fbec665fdcea2e121, entries=150, sequenceid=1151, filesize=12.0 K 2024-11-14T06:18:48,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/a2c152b0203e415c8842a73ebccc44fa as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/a2c152b0203e415c8842a73ebccc44fa 2024-11-14T06:18:48,920 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/a2c152b0203e415c8842a73ebccc44fa, entries=150, sequenceid=1151, filesize=12.0 K 2024-11-14T06:18:48,921 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for d062eac452eae155317450043cfef448 in 952ms, sequenceid=1151, compaction requested=true 2024-11-14T06:18:48,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d062eac452eae155317450043cfef448: 2024-11-14T06:18:48,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d062eac452eae155317450043cfef448:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T06:18:48,921 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:18:48,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:48,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d062eac452eae155317450043cfef448:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T06:18:48,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:48,921 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:18:48,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d062eac452eae155317450043cfef448:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T06:18:48,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:18:48,922 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94447 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:18:48,923 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:18:48,923 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): d062eac452eae155317450043cfef448/A is initiating minor compaction (all files) 2024-11-14T06:18:48,923 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): d062eac452eae155317450043cfef448/B is initiating minor compaction (all files) 2024-11-14T06:18:48,923 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d062eac452eae155317450043cfef448/A in TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:48,923 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d062eac452eae155317450043cfef448/B in TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:48,923 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/6922ec1be880493abf4e32a70992895e, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/f45f4d0293f64208b835f3fd01c82e18, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/38b38e681147441fbec665fdcea2e121] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp, totalSize=36.7 K 2024-11-14T06:18:48,923 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/4993d807f91748ca9434a4faad1d349b, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/4f700b3a47ae4817a59b0265c5599483, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/774faa2243864c2abeefc2f3a13f1187] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp, totalSize=92.2 K 2024-11-14T06:18:48,923 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:48,923 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. files: [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/4993d807f91748ca9434a4faad1d349b, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/4f700b3a47ae4817a59b0265c5599483, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/774faa2243864c2abeefc2f3a13f1187] 2024-11-14T06:18:48,924 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 6922ec1be880493abf4e32a70992895e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=1115, earliestPutTs=1731565126015 2024-11-14T06:18:48,924 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4993d807f91748ca9434a4faad1d349b, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=1115, earliestPutTs=1731565126015 2024-11-14T06:18:48,924 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f700b3a47ae4817a59b0265c5599483, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=1135, earliestPutTs=1731565126678 2024-11-14T06:18:48,924 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting f45f4d0293f64208b835f3fd01c82e18, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1135, earliestPutTs=1731565126678 2024-11-14T06:18:48,924 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 774faa2243864c2abeefc2f3a13f1187, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=1151, earliestPutTs=1731565127334 2024-11-14T06:18:48,924 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 38b38e681147441fbec665fdcea2e121, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1151, earliestPutTs=1731565127334 2024-11-14T06:18:48,942 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d062eac452eae155317450043cfef448#B#compaction#543 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:18:48,942 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/e77d24738e754cf1b50d280dca0d2f01 is 50, key is test_row_0/B:col10/1731565127334/Put/seqid=0 2024-11-14T06:18:48,957 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d062eac452eae155317450043cfef448] 2024-11-14T06:18:48,989 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114913a9e142d664b1e9268cf0e5dca199c_d062eac452eae155317450043cfef448 store=[table=TestAcidGuarantees family=A region=d062eac452eae155317450043cfef448] 2024-11-14T06:18:48,991 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114913a9e142d664b1e9268cf0e5dca199c_d062eac452eae155317450043cfef448, store=[table=TestAcidGuarantees family=A region=d062eac452eae155317450043cfef448] 2024-11-14T06:18:48,991 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114913a9e142d664b1e9268cf0e5dca199c_d062eac452eae155317450043cfef448 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d062eac452eae155317450043cfef448] 2024-11-14T06:18:48,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-14T06:18:48,998 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:18:48,998 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T06:18:49,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-14T06:18:49,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-14T06:18:49,004 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T06:18:49,005 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T06:18:49,005 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T06:18:49,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742469_1645 (size=13085) 2024-11-14T06:18:49,024 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/e77d24738e754cf1b50d280dca0d2f01 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/e77d24738e754cf1b50d280dca0d2f01 2024-11-14T06:18:49,029 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d062eac452eae155317450043cfef448/B of d062eac452eae155317450043cfef448 into e77d24738e754cf1b50d280dca0d2f01(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:18:49,029 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d062eac452eae155317450043cfef448: 2024-11-14T06:18:49,029 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., storeName=d062eac452eae155317450043cfef448/B, priority=13, startTime=1731565128921; duration=0sec 2024-11-14T06:18:49,029 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:18:49,029 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d062eac452eae155317450043cfef448:B 2024-11-14T06:18:49,029 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:18:49,031 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:18:49,031 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): d062eac452eae155317450043cfef448/C is initiating minor compaction (all files) 2024-11-14T06:18:49,031 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d062eac452eae155317450043cfef448/C in TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:49,031 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/d52c18fbf82141c08b05496681bc0b89, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/a90a94b66f644432a37f6498f4d86a51, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/a2c152b0203e415c8842a73ebccc44fa] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp, totalSize=36.7 K 2024-11-14T06:18:49,031 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting d52c18fbf82141c08b05496681bc0b89, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=1115, earliestPutTs=1731565126015 2024-11-14T06:18:49,032 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting a90a94b66f644432a37f6498f4d86a51, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1135, earliestPutTs=1731565126678 2024-11-14T06:18:49,032 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting a2c152b0203e415c8842a73ebccc44fa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1151, earliestPutTs=1731565127334 2024-11-14T06:18:49,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742470_1646 (size=4469) 2024-11-14T06:18:49,041 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d062eac452eae155317450043cfef448#A#compaction#544 average throughput is 0.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:18:49,041 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/32d38ab6ebe541ae8ce3f0c9a1e594a4 is 175, key is test_row_0/A:col10/1731565127334/Put/seqid=0 2024-11-14T06:18:49,054 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d062eac452eae155317450043cfef448#C#compaction#545 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:18:49,054 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/715256f087344d75bff598272ed37d1f is 50, key is test_row_0/C:col10/1731565127334/Put/seqid=0 2024-11-14T06:18:49,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-14T06:18:49,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742471_1647 (size=32039) 2024-11-14T06:18:49,140 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d062eac452eae155317450043cfef448 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-14T06:18:49,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=A 2024-11-14T06:18:49,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:49,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=B 2024-11-14T06:18:49,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:49,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=C 2024-11-14T06:18:49,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:49,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on d062eac452eae155317450043cfef448 2024-11-14T06:18:49,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742472_1648 (size=13085) 2024-11-14T06:18:49,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111424c1b86ea7b4433bbf309fd8a787e9d6_d062eac452eae155317450043cfef448 is 50, key is test_row_0/A:col10/1731565127990/Put/seqid=0 2024-11-14T06:18:49,151 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/715256f087344d75bff598272ed37d1f as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/715256f087344d75bff598272ed37d1f 2024-11-14T06:18:49,158 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d062eac452eae155317450043cfef448/C of d062eac452eae155317450043cfef448 into 715256f087344d75bff598272ed37d1f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:18:49,158 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d062eac452eae155317450043cfef448: 2024-11-14T06:18:49,158 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., storeName=d062eac452eae155317450043cfef448/C, priority=13, startTime=1731565128921; duration=0sec 2024-11-14T06:18:49,158 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:49,158 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d062eac452eae155317450043cfef448:C 2024-11-14T06:18:49,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:49,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52978 deadline: 1731565189154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:49,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:49,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:49,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:49,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:49,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 300 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565189155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:49,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:49,161 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:49,161 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:49,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 277 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565189155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:49,162 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:49,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-14T06:18:49,162 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:49,162 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:49,162 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:49,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:49,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. as already flushing 2024-11-14T06:18:49,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:49,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:49,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:49,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:49,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:49,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 269 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565189161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:49,168 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:49,168 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:49,168 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:49,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:49,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 295 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565189160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:49,168 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:49,168 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:49,168 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:49,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742473_1649 (size=14994) 2024-11-14T06:18:49,209 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:18:49,213 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111424c1b86ea7b4433bbf309fd8a787e9d6_d062eac452eae155317450043cfef448 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111424c1b86ea7b4433bbf309fd8a787e9d6_d062eac452eae155317450043cfef448 2024-11-14T06:18:49,214 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/6db3c5ee34294a179d13fa6e1aa9eaab, store: [table=TestAcidGuarantees family=A region=d062eac452eae155317450043cfef448] 2024-11-14T06:18:49,214 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/6db3c5ee34294a179d13fa6e1aa9eaab is 175, key is test_row_0/A:col10/1731565127990/Put/seqid=0 2024-11-14T06:18:49,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742474_1650 (size=39949) 2024-11-14T06:18:49,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:49,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52978 deadline: 1731565189268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:49,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:49,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:49,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:49,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 278 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565189268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:49,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:49,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:49,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:49,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:49,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:49,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 301 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565189268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:49,272 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:49,272 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:49,272 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:49,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:49,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:49,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 296 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565189277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:49,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 270 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565189277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:49,281 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:49,281 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:49,281 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:49,281 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:49,281 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:49,281 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:49,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-14T06:18:49,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:49,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. as already flushing 2024-11-14T06:18:49,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:49,316 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:49,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:49,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:49,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-14T06:18:49,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-14T06:18:49,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:49,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. as already flushing 2024-11-14T06:18:49,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:49,469 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:49,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:49,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:49,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:49,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52978 deadline: 1731565189477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:49,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:49,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:49,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:49,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:49,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 302 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565189477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:49,481 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:49,481 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:49,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:49,481 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:49,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 279 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565189477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:49,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:49,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:49,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:49,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:49,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 297 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565189488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:49,491 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:49,491 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:49,491 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:49,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:49,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 271 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565189488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:49,491 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:49,491 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:49,491 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:49,522 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/32d38ab6ebe541ae8ce3f0c9a1e594a4 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/32d38ab6ebe541ae8ce3f0c9a1e594a4 2024-11-14T06:18:49,528 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d062eac452eae155317450043cfef448/A of d062eac452eae155317450043cfef448 into 32d38ab6ebe541ae8ce3f0c9a1e594a4(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:18:49,528 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d062eac452eae155317450043cfef448: 2024-11-14T06:18:49,528 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., storeName=d062eac452eae155317450043cfef448/A, priority=13, startTime=1731565128921; duration=0sec 2024-11-14T06:18:49,528 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:49,528 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d062eac452eae155317450043cfef448:A 2024-11-14T06:18:49,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-14T06:18:49,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:49,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. as already flushing 2024-11-14T06:18:49,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:49,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:49,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:49,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:49,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-14T06:18:49,645 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=1171, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/6db3c5ee34294a179d13fa6e1aa9eaab 2024-11-14T06:18:49,650 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/ca0289ba90b64730b3e8cb239fde3598 is 50, key is test_row_0/B:col10/1731565127990/Put/seqid=0 2024-11-14T06:18:49,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742475_1651 (size=12301) 2024-11-14T06:18:49,658 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=1171 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/ca0289ba90b64730b3e8cb239fde3598 2024-11-14T06:18:49,670 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/35c8b3fcd8654aa6aaac74c0a5e5112a is 50, key is test_row_0/C:col10/1731565127990/Put/seqid=0 2024-11-14T06:18:49,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742476_1652 (size=12301) 2024-11-14T06:18:49,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-14T06:18:49,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:49,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. as already flushing 2024-11-14T06:18:49,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:49,775 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:49,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:49,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:49,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:49,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52978 deadline: 1731565189787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:49,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:49,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:49,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:49,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 280 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565189788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:49,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:49,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:49,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:49,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:49,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:49,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 303 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565189788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:49,791 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:49,791 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:49,791 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:49,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:49,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 298 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565189797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:49,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:49,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 272 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565189797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:49,799 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:49,799 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:49,799 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:49,799 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:49,799 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:49,799 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:49,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-14T06:18:49,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:49,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. as already flushing 2024-11-14T06:18:49,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:49,928 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:49,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:49,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:50,075 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=1171 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/35c8b3fcd8654aa6aaac74c0a5e5112a 2024-11-14T06:18:50,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/6db3c5ee34294a179d13fa6e1aa9eaab as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/6db3c5ee34294a179d13fa6e1aa9eaab 2024-11-14T06:18:50,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-14T06:18:50,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:50,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. as already flushing 2024-11-14T06:18:50,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:50,081 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:50,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:50,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/6db3c5ee34294a179d13fa6e1aa9eaab, entries=200, sequenceid=1171, filesize=39.0 K 2024-11-14T06:18:50,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:50,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/ca0289ba90b64730b3e8cb239fde3598 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/ca0289ba90b64730b3e8cb239fde3598 2024-11-14T06:18:50,086 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/ca0289ba90b64730b3e8cb239fde3598, entries=150, sequenceid=1171, filesize=12.0 K 2024-11-14T06:18:50,087 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/35c8b3fcd8654aa6aaac74c0a5e5112a as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/35c8b3fcd8654aa6aaac74c0a5e5112a 2024-11-14T06:18:50,091 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/35c8b3fcd8654aa6aaac74c0a5e5112a, entries=150, sequenceid=1171, filesize=12.0 K 2024-11-14T06:18:50,092 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for d062eac452eae155317450043cfef448 in 952ms, sequenceid=1171, compaction requested=false 2024-11-14T06:18:50,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d062eac452eae155317450043cfef448: 2024-11-14T06:18:50,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-14T06:18:50,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-14T06:18:50,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:50,234 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing d062eac452eae155317450043cfef448 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-14T06:18:50,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=A 2024-11-14T06:18:50,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:50,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=B 2024-11-14T06:18:50,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:50,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=C 2024-11-14T06:18:50,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:50,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411146158c39b20b54559a523e1923c6ee94c_d062eac452eae155317450043cfef448 is 50, key is test_row_0/A:col10/1731565129159/Put/seqid=0 2024-11-14T06:18:50,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742477_1653 (size=12454) 2024-11-14T06:18:50,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:18:50,254 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411146158c39b20b54559a523e1923c6ee94c_d062eac452eae155317450043cfef448 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411146158c39b20b54559a523e1923c6ee94c_d062eac452eae155317450043cfef448 2024-11-14T06:18:50,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/6fec64f49e0f4eb0aadd833de837bb53, store: [table=TestAcidGuarantees family=A region=d062eac452eae155317450043cfef448] 2024-11-14T06:18:50,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/6fec64f49e0f4eb0aadd833de837bb53 is 175, key is test_row_0/A:col10/1731565129159/Put/seqid=0 2024-11-14T06:18:50,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742478_1654 (size=31255) 2024-11-14T06:18:50,263 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=1188, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/6fec64f49e0f4eb0aadd833de837bb53 2024-11-14T06:18:50,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/e8a814fb3dc44a9881dda7f24ba9df8c is 50, key is test_row_0/B:col10/1731565129159/Put/seqid=0 2024-11-14T06:18:50,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on d062eac452eae155317450043cfef448 2024-11-14T06:18:50,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. as already flushing 2024-11-14T06:18:50,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742479_1655 (size=12301) 2024-11-14T06:18:50,316 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=1188 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/e8a814fb3dc44a9881dda7f24ba9df8c 2024-11-14T06:18:50,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:50,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 300 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565190317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:50,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:50,325 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:50,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 274 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565190318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:50,325 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:50,325 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:50,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:50,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 308 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565190319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:50,325 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:50,325 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:50,325 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:50,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:50,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52978 deadline: 1731565190322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:50,326 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:50,326 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:50,326 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:50,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:50,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 285 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565190322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:50,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:50,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:50,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:50,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:50,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:50,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:50,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/7e2dc93eaca74776bfed09ef83889686 is 50, key is test_row_0/C:col10/1731565129159/Put/seqid=0 2024-11-14T06:18:50,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742480_1656 (size=12301) 2024-11-14T06:18:50,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:50,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 301 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565190427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:50,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:50,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 275 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565190427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:50,431 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:50,431 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:50,431 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:50,431 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:50,431 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:50,431 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:50,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:50,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52978 deadline: 1731565190443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:50,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:50,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 286 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565190443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:50,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:50,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:50,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:50,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:50,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 309 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565190443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:50,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:50,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:50,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:50,446 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:50,446 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:50,446 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:50,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:50,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 302 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565190637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:50,640 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:50,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:50,640 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:50,640 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:50,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 276 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565190637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:50,640 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:50,640 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:50,640 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:50,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:50,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52978 deadline: 1731565190657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:50,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:50,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:50,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:50,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:50,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 287 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565190658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:50,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:50,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:50,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:50,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:50,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 310 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565190658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:50,662 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:50,662 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:50,662 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:50,742 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=1188 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/7e2dc93eaca74776bfed09ef83889686 2024-11-14T06:18:50,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/6fec64f49e0f4eb0aadd833de837bb53 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/6fec64f49e0f4eb0aadd833de837bb53 2024-11-14T06:18:50,751 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/6fec64f49e0f4eb0aadd833de837bb53, entries=150, sequenceid=1188, filesize=30.5 K 2024-11-14T06:18:50,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/e8a814fb3dc44a9881dda7f24ba9df8c as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/e8a814fb3dc44a9881dda7f24ba9df8c 2024-11-14T06:18:50,756 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/e8a814fb3dc44a9881dda7f24ba9df8c, entries=150, sequenceid=1188, filesize=12.0 K 2024-11-14T06:18:50,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/7e2dc93eaca74776bfed09ef83889686 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/7e2dc93eaca74776bfed09ef83889686 2024-11-14T06:18:50,761 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/7e2dc93eaca74776bfed09ef83889686, entries=150, sequenceid=1188, filesize=12.0 K 2024-11-14T06:18:50,761 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for d062eac452eae155317450043cfef448 in 527ms, sequenceid=1188, compaction requested=true 2024-11-14T06:18:50,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for d062eac452eae155317450043cfef448: 2024-11-14T06:18:50,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:50,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-14T06:18:50,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-14T06:18:50,765 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-14T06:18:50,765 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7580 sec 2024-11-14T06:18:50,767 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.7670 sec 2024-11-14T06:18:50,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on d062eac452eae155317450043cfef448 2024-11-14T06:18:50,954 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d062eac452eae155317450043cfef448 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-14T06:18:50,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=A 2024-11-14T06:18:50,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:50,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=B 2024-11-14T06:18:50,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:50,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=C 2024-11-14T06:18:50,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:50,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:50,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52978 deadline: 1731565190974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:50,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:50,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:50,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:50,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:50,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 289 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565190975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:50,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:50,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:50,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:50,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:50,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 281 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565190975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:50,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:50,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 312 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565190977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:50,983 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:50,983 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:50,983 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:50,984 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:50,984 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:50,984 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:50,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:50,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 308 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565190981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:50,985 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:50,985 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:50,985 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:50,991 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114cd971e1750f848c286f3acef47be3126_d062eac452eae155317450043cfef448 is 50, key is test_row_0/A:col10/1731565130950/Put/seqid=0 2024-11-14T06:18:51,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742481_1657 (size=12454) 2024-11-14T06:18:51,020 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:18:51,024 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114cd971e1750f848c286f3acef47be3126_d062eac452eae155317450043cfef448 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114cd971e1750f848c286f3acef47be3126_d062eac452eae155317450043cfef448 2024-11-14T06:18:51,024 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/325127a356c34f01863d503240d9f964, store: [table=TestAcidGuarantees family=A region=d062eac452eae155317450043cfef448] 2024-11-14T06:18:51,025 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/325127a356c34f01863d503240d9f964 is 175, key is test_row_0/A:col10/1731565130950/Put/seqid=0 2024-11-14T06:18:51,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742482_1658 (size=31255) 2024-11-14T06:18:51,028 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=1207, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/325127a356c34f01863d503240d9f964 2024-11-14T06:18:51,035 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/2ab5b879d2324e3d8962dbd2cd154c7b is 50, key is test_row_0/B:col10/1731565130950/Put/seqid=0 2024-11-14T06:18:51,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742483_1659 (size=12301) 2024-11-14T06:18:51,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:51,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52978 deadline: 1731565191087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:51,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:51,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 290 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565191087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:51,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:51,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:51,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 282 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565191087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:51,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:51,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:51,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:51,093 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:51,093 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:51,093 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:51,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 309 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565191088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:51,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:51,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:51,093 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:51,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:51,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:51,093 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:51,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 313 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565191088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:51,093 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:51,094 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:51,094 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:51,094 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:51,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-14T06:18:51,148 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:18:51,148 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T06:18:51,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-14T06:18:51,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-14T06:18:51,149 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T06:18:51,150 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T06:18:51,150 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T06:18:51,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-14T06:18:51,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:51,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 291 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565191297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:51,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:51,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:51,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:51,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52978 deadline: 1731565191297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:51,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:51,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:51,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:51,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:51,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 314 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565191298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:51,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:51,302 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:51,302 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:51,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:51,302 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:51,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 310 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565191298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:51,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:51,302 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:51,302 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:51,302 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:51,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 283 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565191298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:51,302 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:51,302 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:51,302 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:51,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-14T06:18:51,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:51,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. as already flushing 2024-11-14T06:18:51,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:51,303 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:51,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:51,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:51,456 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-14T06:18:51,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:51,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. as already flushing 2024-11-14T06:18:51,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:51,457 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:51,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:51,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:51,459 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=1207 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/2ab5b879d2324e3d8962dbd2cd154c7b 2024-11-14T06:18:51,467 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/c4442dc47652491489eb392d6e6e4456 is 50, key is test_row_0/C:col10/1731565130950/Put/seqid=0 2024-11-14T06:18:51,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-14T06:18:51,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742484_1660 (size=12301) 2024-11-14T06:18:51,476 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=1207 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/c4442dc47652491489eb392d6e6e4456 2024-11-14T06:18:51,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/325127a356c34f01863d503240d9f964 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/325127a356c34f01863d503240d9f964 2024-11-14T06:18:51,483 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/325127a356c34f01863d503240d9f964, entries=150, sequenceid=1207, filesize=30.5 K 2024-11-14T06:18:51,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/2ab5b879d2324e3d8962dbd2cd154c7b as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/2ab5b879d2324e3d8962dbd2cd154c7b 2024-11-14T06:18:51,487 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/2ab5b879d2324e3d8962dbd2cd154c7b, entries=150, sequenceid=1207, filesize=12.0 K 2024-11-14T06:18:51,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/c4442dc47652491489eb392d6e6e4456 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/c4442dc47652491489eb392d6e6e4456 2024-11-14T06:18:51,492 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/c4442dc47652491489eb392d6e6e4456, entries=150, sequenceid=1207, filesize=12.0 K 2024-11-14T06:18:51,492 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for d062eac452eae155317450043cfef448 in 538ms, sequenceid=1207, compaction requested=true 2024-11-14T06:18:51,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d062eac452eae155317450043cfef448: 2024-11-14T06:18:51,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d062eac452eae155317450043cfef448:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T06:18:51,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:51,493 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T06:18:51,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d062eac452eae155317450043cfef448:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T06:18:51,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:51,493 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T06:18:51,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d062eac452eae155317450043cfef448:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T06:18:51,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:18:51,494 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T06:18:51,494 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134498 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T06:18:51,494 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): d062eac452eae155317450043cfef448/A is initiating minor compaction (all files) 2024-11-14T06:18:51,494 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): d062eac452eae155317450043cfef448/B is initiating minor compaction (all files) 2024-11-14T06:18:51,494 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d062eac452eae155317450043cfef448/B in TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:51,494 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d062eac452eae155317450043cfef448/A in TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:51,494 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/32d38ab6ebe541ae8ce3f0c9a1e594a4, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/6db3c5ee34294a179d13fa6e1aa9eaab, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/6fec64f49e0f4eb0aadd833de837bb53, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/325127a356c34f01863d503240d9f964] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp, totalSize=131.3 K 2024-11-14T06:18:51,494 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/e77d24738e754cf1b50d280dca0d2f01, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/ca0289ba90b64730b3e8cb239fde3598, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/e8a814fb3dc44a9881dda7f24ba9df8c, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/2ab5b879d2324e3d8962dbd2cd154c7b] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp, totalSize=48.8 K 2024-11-14T06:18:51,494 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:51,494 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. files: [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/32d38ab6ebe541ae8ce3f0c9a1e594a4, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/6db3c5ee34294a179d13fa6e1aa9eaab, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/6fec64f49e0f4eb0aadd833de837bb53, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/325127a356c34f01863d503240d9f964] 2024-11-14T06:18:51,494 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting e77d24738e754cf1b50d280dca0d2f01, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=1151, earliestPutTs=1731565127334 2024-11-14T06:18:51,494 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32d38ab6ebe541ae8ce3f0c9a1e594a4, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=1151, earliestPutTs=1731565127334 2024-11-14T06:18:51,494 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting ca0289ba90b64730b3e8cb239fde3598, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1171, earliestPutTs=1731565127987 2024-11-14T06:18:51,494 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6db3c5ee34294a179d13fa6e1aa9eaab, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=1171, earliestPutTs=1731565127987 2024-11-14T06:18:51,494 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting e8a814fb3dc44a9881dda7f24ba9df8c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1188, earliestPutTs=1731565129147 2024-11-14T06:18:51,495 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6fec64f49e0f4eb0aadd833de837bb53, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=1188, earliestPutTs=1731565129147 2024-11-14T06:18:51,495 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ab5b879d2324e3d8962dbd2cd154c7b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1207, earliestPutTs=1731565130318 2024-11-14T06:18:51,495 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 325127a356c34f01863d503240d9f964, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=1207, earliestPutTs=1731565130318 2024-11-14T06:18:51,513 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d062eac452eae155317450043cfef448#B#compaction#555 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:18:51,514 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/b080ce8739ac479d8e5cd66ca6be61df is 50, key is test_row_0/B:col10/1731565130950/Put/seqid=0 2024-11-14T06:18:51,516 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d062eac452eae155317450043cfef448] 2024-11-14T06:18:51,525 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114efcbf5dca80b4bf6b16e6f1487679761_d062eac452eae155317450043cfef448 store=[table=TestAcidGuarantees family=A region=d062eac452eae155317450043cfef448] 2024-11-14T06:18:51,526 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114efcbf5dca80b4bf6b16e6f1487679761_d062eac452eae155317450043cfef448, store=[table=TestAcidGuarantees family=A region=d062eac452eae155317450043cfef448] 2024-11-14T06:18:51,527 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114efcbf5dca80b4bf6b16e6f1487679761_d062eac452eae155317450043cfef448 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d062eac452eae155317450043cfef448] 2024-11-14T06:18:51,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742485_1661 (size=13221) 2024-11-14T06:18:51,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742486_1662 (size=4469) 2024-11-14T06:18:51,552 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d062eac452eae155317450043cfef448#A#compaction#556 average throughput is 0.70 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:18:51,552 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/6268ab67e7984700a19f1354e7fcedad is 175, key is test_row_0/A:col10/1731565130950/Put/seqid=0 2024-11-14T06:18:51,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742487_1663 (size=32175) 2024-11-14T06:18:51,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-14T06:18:51,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:51,609 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing d062eac452eae155317450043cfef448 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-14T06:18:51,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on d062eac452eae155317450043cfef448 2024-11-14T06:18:51,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=A 2024-11-14T06:18:51,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:51,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=B 2024-11-14T06:18:51,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:51,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=C 2024-11-14T06:18:51,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:51,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. as already flushing 2024-11-14T06:18:51,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114caf8de77b8f84fd2b1a41bdeab2050da_d062eac452eae155317450043cfef448 is 50, key is test_row_0/A:col10/1731565131609/Put/seqid=0 2024-11-14T06:18:51,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:51,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 295 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565191625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:51,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:51,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:51,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:51,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52978 deadline: 1731565191625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:51,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:51,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:51,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:51,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:51,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 314 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565191628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:51,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:51,632 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:51,632 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:51,632 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:51,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742488_1664 (size=14994) 2024-11-14T06:18:51,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:51,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 287 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565191629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:51,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:51,637 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:51,637 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:51,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 319 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565191631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:51,637 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:51,638 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:51,638 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:51,638 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:51,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:51,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 296 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565191737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:51,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:51,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52978 deadline: 1731565191737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:51,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:51,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:51,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:51,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:51,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 315 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565191737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:51,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:51,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:51,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:51,742 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:51,742 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:51,742 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:51,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:51,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 288 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565191748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:51,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:51,751 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:51,751 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:51,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 320 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565191748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:51,751 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:51,751 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:51,751 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:51,751 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:51,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-14T06:18:51,938 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/b080ce8739ac479d8e5cd66ca6be61df as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/b080ce8739ac479d8e5cd66ca6be61df 2024-11-14T06:18:51,943 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d062eac452eae155317450043cfef448/B of d062eac452eae155317450043cfef448 into b080ce8739ac479d8e5cd66ca6be61df(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:18:51,943 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d062eac452eae155317450043cfef448: 2024-11-14T06:18:51,943 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., storeName=d062eac452eae155317450043cfef448/B, priority=12, startTime=1731565131493; duration=0sec 2024-11-14T06:18:51,943 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:18:51,943 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d062eac452eae155317450043cfef448:B 2024-11-14T06:18:51,943 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T06:18:51,944 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T06:18:51,944 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): d062eac452eae155317450043cfef448/C is initiating minor compaction (all files) 2024-11-14T06:18:51,944 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d062eac452eae155317450043cfef448/C in TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:51,944 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/715256f087344d75bff598272ed37d1f, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/35c8b3fcd8654aa6aaac74c0a5e5112a, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/7e2dc93eaca74776bfed09ef83889686, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/c4442dc47652491489eb392d6e6e4456] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp, totalSize=48.8 K 2024-11-14T06:18:51,945 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 715256f087344d75bff598272ed37d1f, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=1151, earliestPutTs=1731565127334 2024-11-14T06:18:51,946 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 35c8b3fcd8654aa6aaac74c0a5e5112a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1171, earliestPutTs=1731565127987 2024-11-14T06:18:51,946 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e2dc93eaca74776bfed09ef83889686, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1188, earliestPutTs=1731565129147 2024-11-14T06:18:51,946 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting c4442dc47652491489eb392d6e6e4456, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1207, earliestPutTs=1731565130318 2024-11-14T06:18:51,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:51,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52978 deadline: 1731565191947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:51,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:51,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:51,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:51,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:51,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 316 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565191947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:51,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:51,953 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:51,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 297 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565191947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:51,953 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:51,953 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:51,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:51,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:51,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:51,959 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d062eac452eae155317450043cfef448#C#compaction#558 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:18:51,959 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/0b24e0b4f4b847dd8fad2a64ab419831 is 50, key is test_row_0/C:col10/1731565130950/Put/seqid=0 2024-11-14T06:18:51,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:51,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 289 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565191957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:51,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:51,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 321 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565191957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:51,964 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:51,964 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:51,964 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:51,964 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:51,964 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:51,964 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:51,967 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/6268ab67e7984700a19f1354e7fcedad as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/6268ab67e7984700a19f1354e7fcedad 2024-11-14T06:18:51,971 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d062eac452eae155317450043cfef448/A of d062eac452eae155317450043cfef448 into 6268ab67e7984700a19f1354e7fcedad(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:18:51,971 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d062eac452eae155317450043cfef448: 2024-11-14T06:18:51,971 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., storeName=d062eac452eae155317450043cfef448/A, priority=12, startTime=1731565131493; duration=0sec 2024-11-14T06:18:51,971 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:51,971 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d062eac452eae155317450043cfef448:A 2024-11-14T06:18:51,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742489_1665 (size=13221) 2024-11-14T06:18:51,994 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/0b24e0b4f4b847dd8fad2a64ab419831 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/0b24e0b4f4b847dd8fad2a64ab419831 2024-11-14T06:18:52,004 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d062eac452eae155317450043cfef448/C of d062eac452eae155317450043cfef448 into 0b24e0b4f4b847dd8fad2a64ab419831(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:18:52,004 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d062eac452eae155317450043cfef448: 2024-11-14T06:18:52,004 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., storeName=d062eac452eae155317450043cfef448/C, priority=12, startTime=1731565131493; duration=0sec 2024-11-14T06:18:52,005 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:52,005 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d062eac452eae155317450043cfef448:C 2024-11-14T06:18:52,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:18:52,038 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114caf8de77b8f84fd2b1a41bdeab2050da_d062eac452eae155317450043cfef448 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114caf8de77b8f84fd2b1a41bdeab2050da_d062eac452eae155317450043cfef448 2024-11-14T06:18:52,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/26f6d3c5cab14d5f8209d83f3be8bea9, store: [table=TestAcidGuarantees family=A region=d062eac452eae155317450043cfef448] 2024-11-14T06:18:52,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/26f6d3c5cab14d5f8209d83f3be8bea9 is 175, key is test_row_0/A:col10/1731565131609/Put/seqid=0 2024-11-14T06:18:52,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742490_1666 (size=39949) 2024-11-14T06:18:52,050 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=1221, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/26f6d3c5cab14d5f8209d83f3be8bea9 2024-11-14T06:18:52,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/e81fb802c7844d00b1c81e2f2b333c7b is 50, key is test_row_0/B:col10/1731565131609/Put/seqid=0 2024-11-14T06:18:52,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742491_1667 (size=12301) 2024-11-14T06:18:52,062 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=1221 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/e81fb802c7844d00b1c81e2f2b333c7b 2024-11-14T06:18:52,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/d73e0d5027cf4aab84d6e4f1c2ce133d is 50, key is test_row_0/C:col10/1731565131609/Put/seqid=0 2024-11-14T06:18:52,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742492_1668 (size=12301) 2024-11-14T06:18:52,083 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=1221 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/d73e0d5027cf4aab84d6e4f1c2ce133d 2024-11-14T06:18:52,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/26f6d3c5cab14d5f8209d83f3be8bea9 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/26f6d3c5cab14d5f8209d83f3be8bea9 2024-11-14T06:18:52,098 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/26f6d3c5cab14d5f8209d83f3be8bea9, entries=200, sequenceid=1221, filesize=39.0 K 2024-11-14T06:18:52,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/e81fb802c7844d00b1c81e2f2b333c7b as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/e81fb802c7844d00b1c81e2f2b333c7b 2024-11-14T06:18:52,108 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/e81fb802c7844d00b1c81e2f2b333c7b, entries=150, sequenceid=1221, filesize=12.0 K 2024-11-14T06:18:52,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/d73e0d5027cf4aab84d6e4f1c2ce133d as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/d73e0d5027cf4aab84d6e4f1c2ce133d 2024-11-14T06:18:52,118 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/d73e0d5027cf4aab84d6e4f1c2ce133d, entries=150, sequenceid=1221, filesize=12.0 K 2024-11-14T06:18:52,119 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for d062eac452eae155317450043cfef448 in 510ms, sequenceid=1221, compaction requested=false 2024-11-14T06:18:52,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for d062eac452eae155317450043cfef448: 2024-11-14T06:18:52,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:52,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-14T06:18:52,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-14T06:18:52,122 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-14T06:18:52,122 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 970 msec 2024-11-14T06:18:52,124 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 975 msec 2024-11-14T06:18:52,262 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d062eac452eae155317450043cfef448 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-14T06:18:52,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=A 2024-11-14T06:18:52,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:52,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=B 2024-11-14T06:18:52,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:52,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=C 2024-11-14T06:18:52,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:52,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on d062eac452eae155317450043cfef448 2024-11-14T06:18:52,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-14T06:18:52,287 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:18:52,287 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T06:18:52,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-14T06:18:52,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-14T06:18:52,291 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T06:18:52,294 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114aba46c063f8843b0b02e285da7f52b18_d062eac452eae155317450043cfef448 is 50, key is test_row_0/A:col10/1731565131625/Put/seqid=0 2024-11-14T06:18:52,295 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T06:18:52,296 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T06:18:52,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:52,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 323 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565192291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:52,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:52,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 251 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52978 deadline: 1731565192296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:52,305 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:52,305 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:52,305 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:52,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:52,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742493_1669 (size=12454) 2024-11-14T06:18:52,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:52,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:52,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:52,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:52,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 320 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565192304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:52,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 292 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565192303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:52,313 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:52,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:52,313 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:52,313 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:52,313 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:52,313 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:52,313 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:52,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 301 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565192305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:52,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:52,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:52,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:52,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-14T06:18:52,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:52,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 324 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565192407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:52,410 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:52,410 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:52,410 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:52,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:52,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52978 deadline: 1731565192417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:52,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:52,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:52,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:52,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 321 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565192417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:52,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:52,423 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:52,423 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:52,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:52,423 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:52,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 302 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565192417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:52,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:52,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:52,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:52,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:52,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 293 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565192417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:52,424 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:52,424 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:52,424 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:52,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-14T06:18:52,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:52,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. as already flushing 2024-11-14T06:18:52,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:52,453 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:52,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:52,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:52,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-14T06:18:52,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:52,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. as already flushing 2024-11-14T06:18:52,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:52,606 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:52,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:52,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-14T06:18:52,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:52,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:52,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 325 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565192617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:52,622 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:52,622 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:52,622 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:52,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:52,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52978 deadline: 1731565192628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:52,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:52,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:52,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:52,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:52,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 303 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565192628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:52,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:52,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:52,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:52,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:52,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 322 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565192628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:52,631 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:52,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:52,632 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:52,632 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:52,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 294 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565192629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:52,632 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:52,632 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:52,632 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:52,714 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:18:52,717 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114aba46c063f8843b0b02e285da7f52b18_d062eac452eae155317450043cfef448 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114aba46c063f8843b0b02e285da7f52b18_d062eac452eae155317450043cfef448 2024-11-14T06:18:52,719 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/92573a53ca23415990acb308ce53470d, store: [table=TestAcidGuarantees family=A region=d062eac452eae155317450043cfef448] 2024-11-14T06:18:52,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/92573a53ca23415990acb308ce53470d is 175, key is test_row_0/A:col10/1731565131625/Put/seqid=0 2024-11-14T06:18:52,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742494_1670 (size=31255) 2024-11-14T06:18:52,740 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=1243, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/92573a53ca23415990acb308ce53470d 2024-11-14T06:18:52,760 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/cf2326998c194b179a5cacbc75671a2d is 50, key is test_row_0/B:col10/1731565131625/Put/seqid=0 2024-11-14T06:18:52,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-14T06:18:52,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:52,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. as already flushing 2024-11-14T06:18:52,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:52,760 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:52,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:52,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:52,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742495_1671 (size=12301) 2024-11-14T06:18:52,804 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=1243 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/cf2326998c194b179a5cacbc75671a2d 2024-11-14T06:18:52,813 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/fc47320ccf9f4dd5997bc34aa0bb9c3e is 50, key is test_row_0/C:col10/1731565131625/Put/seqid=0 2024-11-14T06:18:52,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742496_1672 (size=12301) 2024-11-14T06:18:52,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-14T06:18:52,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-14T06:18:52,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:52,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. as already flushing 2024-11-14T06:18:52,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:52,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:52,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:52,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:52,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:52,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 326 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53048 deadline: 1731565192927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:52,931 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:52,931 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:52,931 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:52,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:52,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 304 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1731565192937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:52,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:52,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:52,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:52,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:52,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52978 deadline: 1731565192937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:52,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:52,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:52,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:52,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 323 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53070 deadline: 1731565192937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:52,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:52,942 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:52,942 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:52,942 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:52,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T06:18:52,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] ipc.CallRunner(138): callId: 295 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53060 deadline: 1731565192937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:52,942 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 , the old value is region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:52,942 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T06:18:52,942 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., hostname=ef93ef3a83c5,42283,1731564965945, seqNum=5 because the exception is null or not the one we care about 2024-11-14T06:18:53,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-14T06:18:53,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:53,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. as already flushing 2024-11-14T06:18:53,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:53,074 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:53,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:53,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:53,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-14T06:18:53,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:53,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. as already flushing 2024-11-14T06:18:53,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:53,227 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:53,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:53,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T06:18:53,250 INFO [Thread-2469 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2469. 2024-11-14T06:18:53,250 DEBUG [Thread-2469 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T06:18:53,250 DEBUG [Thread-2469 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:53,251 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:18:53,251 DEBUG [Thread-2469 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:53,252 INFO [Thread-2461 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2461. 2024-11-14T06:18:53,252 DEBUG [Thread-2461 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T06:18:53,252 DEBUG [Thread-2461 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:53,252 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:18:53,252 DEBUG [Thread-2461 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:53,253 INFO [Thread-2467 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2467. 2024-11-14T06:18:53,253 DEBUG [Thread-2467 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T06:18:53,253 DEBUG [Thread-2467 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:53,253 DEBUG [Thread-2467 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:53,253 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:18:53,255 INFO [Thread-2463 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2463. 2024-11-14T06:18:53,255 DEBUG [Thread-2463 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T06:18:53,255 DEBUG [Thread-2463 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:53,255 DEBUG [Thread-2463 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:53,255 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:18:53,256 INFO [Thread-2465 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2465. 2024-11-14T06:18:53,257 DEBUG [Thread-2465 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T06:18:53,257 DEBUG [Thread-2465 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:53,257 DEBUG [Thread-2465 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:53,257 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:18:53,284 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=1243 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/fc47320ccf9f4dd5997bc34aa0bb9c3e 2024-11-14T06:18:53,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/92573a53ca23415990acb308ce53470d as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/92573a53ca23415990acb308ce53470d 2024-11-14T06:18:53,291 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/92573a53ca23415990acb308ce53470d, entries=150, sequenceid=1243, filesize=30.5 K 2024-11-14T06:18:53,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/cf2326998c194b179a5cacbc75671a2d as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/cf2326998c194b179a5cacbc75671a2d 2024-11-14T06:18:53,294 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/cf2326998c194b179a5cacbc75671a2d, entries=150, sequenceid=1243, filesize=12.0 K 2024-11-14T06:18:53,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/fc47320ccf9f4dd5997bc34aa0bb9c3e as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/fc47320ccf9f4dd5997bc34aa0bb9c3e 2024-11-14T06:18:53,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/fc47320ccf9f4dd5997bc34aa0bb9c3e, entries=150, sequenceid=1243, filesize=12.0 K 2024-11-14T06:18:53,297 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for d062eac452eae155317450043cfef448 in 1035ms, sequenceid=1243, compaction requested=true 2024-11-14T06:18:53,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d062eac452eae155317450043cfef448: 2024-11-14T06:18:53,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d062eac452eae155317450043cfef448:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T06:18:53,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:53,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d062eac452eae155317450043cfef448:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T06:18:53,297 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:18:53,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:53,297 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:18:53,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d062eac452eae155317450043cfef448:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T06:18:53,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:18:53,298 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103379 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:18:53,298 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:18:53,298 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1541): d062eac452eae155317450043cfef448/A is initiating minor compaction (all files) 2024-11-14T06:18:53,298 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): d062eac452eae155317450043cfef448/B is initiating minor compaction (all files) 2024-11-14T06:18:53,298 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d062eac452eae155317450043cfef448/A in TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:53,298 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d062eac452eae155317450043cfef448/B in TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:53,298 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/6268ab67e7984700a19f1354e7fcedad, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/26f6d3c5cab14d5f8209d83f3be8bea9, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/92573a53ca23415990acb308ce53470d] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp, totalSize=101.0 K 2024-11-14T06:18:53,298 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/b080ce8739ac479d8e5cd66ca6be61df, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/e81fb802c7844d00b1c81e2f2b333c7b, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/cf2326998c194b179a5cacbc75671a2d] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp, totalSize=36.9 K 2024-11-14T06:18:53,298 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:53,298 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. files: [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/6268ab67e7984700a19f1354e7fcedad, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/26f6d3c5cab14d5f8209d83f3be8bea9, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/92573a53ca23415990acb308ce53470d] 2024-11-14T06:18:53,298 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting b080ce8739ac479d8e5cd66ca6be61df, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=1207, earliestPutTs=1731565130318 2024-11-14T06:18:53,298 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6268ab67e7984700a19f1354e7fcedad, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=1207, earliestPutTs=1731565130318 2024-11-14T06:18:53,298 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting e81fb802c7844d00b1c81e2f2b333c7b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1221, earliestPutTs=1731565130960 2024-11-14T06:18:53,298 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26f6d3c5cab14d5f8209d83f3be8bea9, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=1221, earliestPutTs=1731565130960 2024-11-14T06:18:53,299 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting cf2326998c194b179a5cacbc75671a2d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1243, earliestPutTs=1731565131625 2024-11-14T06:18:53,299 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92573a53ca23415990acb308ce53470d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=1243, earliestPutTs=1731565131625 2024-11-14T06:18:53,303 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d062eac452eae155317450043cfef448] 2024-11-14T06:18:53,304 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d062eac452eae155317450043cfef448#B#compaction#564 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:18:53,304 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/d4fce2333310452a94c5ca14e9a9b155 is 50, key is test_row_0/B:col10/1731565131625/Put/seqid=0 2024-11-14T06:18:53,305 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114a2f277b066134ec8988bde533fbabf2d_d062eac452eae155317450043cfef448 store=[table=TestAcidGuarantees family=A region=d062eac452eae155317450043cfef448] 2024-11-14T06:18:53,307 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114a2f277b066134ec8988bde533fbabf2d_d062eac452eae155317450043cfef448, store=[table=TestAcidGuarantees family=A region=d062eac452eae155317450043cfef448] 2024-11-14T06:18:53,307 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114a2f277b066134ec8988bde533fbabf2d_d062eac452eae155317450043cfef448 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d062eac452eae155317450043cfef448] 2024-11-14T06:18:53,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742497_1673 (size=4469) 2024-11-14T06:18:53,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742498_1674 (size=13323) 2024-11-14T06:18:53,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-14T06:18:53,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:53,380 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing d062eac452eae155317450043cfef448 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-14T06:18:53,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=A 2024-11-14T06:18:53,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:53,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=B 2024-11-14T06:18:53,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:53,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=C 2024-11-14T06:18:53,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:53,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111426f42d3b6b624b15ab313d3091986d42_d062eac452eae155317450043cfef448 is 50, key is test_row_0/A:col10/1731565132282/Put/seqid=0 2024-11-14T06:18:53,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742499_1675 (size=12454) 2024-11-14T06:18:53,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-14T06:18:53,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42283 {}] regionserver.HRegion(8855): Flush requested on d062eac452eae155317450043cfef448 2024-11-14T06:18:53,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. as already flushing 2024-11-14T06:18:53,438 INFO [Thread-2454 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2454. 2024-11-14T06:18:53,438 DEBUG [Thread-2454 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T06:18:53,438 DEBUG [Thread-2454 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:53,438 DEBUG [Thread-2454 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:53,438 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:18:53,448 INFO [Thread-2450 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2450. 2024-11-14T06:18:53,448 INFO [Thread-2458 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2458. 2024-11-14T06:18:53,448 INFO [Thread-2452 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2452. 2024-11-14T06:18:53,448 DEBUG [Thread-2450 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T06:18:53,448 DEBUG [Thread-2458 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T06:18:53,448 DEBUG [Thread-2452 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T06:18:53,448 DEBUG [Thread-2450 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:53,449 DEBUG [Thread-2458 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:53,449 DEBUG [Thread-2452 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:53,449 DEBUG [Thread-2458 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:53,449 DEBUG [Thread-2450 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:53,449 DEBUG [Thread-2452 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:53,449 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:18:53,449 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:18:53,449 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:18:53,450 INFO [Thread-2456 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2456. 2024-11-14T06:18:53,450 DEBUG [Thread-2456 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T06:18:53,450 DEBUG [Thread-2456 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:53,450 DEBUG [Thread-2456 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:53,450 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:18:53,713 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d062eac452eae155317450043cfef448#A#compaction#565 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:18:53,713 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/4f1044113e4748439920bcb52f91351c is 175, key is test_row_0/A:col10/1731565131625/Put/seqid=0 2024-11-14T06:18:53,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742500_1676 (size=32277) 2024-11-14T06:18:53,721 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/d4fce2333310452a94c5ca14e9a9b155 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/d4fce2333310452a94c5ca14e9a9b155 2024-11-14T06:18:53,725 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d062eac452eae155317450043cfef448/B of d062eac452eae155317450043cfef448 into d4fce2333310452a94c5ca14e9a9b155(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:18:53,725 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d062eac452eae155317450043cfef448: 2024-11-14T06:18:53,725 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., storeName=d062eac452eae155317450043cfef448/B, priority=13, startTime=1731565133297; duration=0sec 2024-11-14T06:18:53,725 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T06:18:53,725 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d062eac452eae155317450043cfef448:B 2024-11-14T06:18:53,725 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T06:18:53,726 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T06:18:53,726 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1541): d062eac452eae155317450043cfef448/C is initiating minor compaction (all files) 2024-11-14T06:18:53,726 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d062eac452eae155317450043cfef448/C in TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:53,726 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/0b24e0b4f4b847dd8fad2a64ab419831, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/d73e0d5027cf4aab84d6e4f1c2ce133d, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/fc47320ccf9f4dd5997bc34aa0bb9c3e] into tmpdir=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp, totalSize=36.9 K 2024-11-14T06:18:53,726 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b24e0b4f4b847dd8fad2a64ab419831, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=1207, earliestPutTs=1731565130318 2024-11-14T06:18:53,726 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting d73e0d5027cf4aab84d6e4f1c2ce133d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1221, earliestPutTs=1731565130960 2024-11-14T06:18:53,727 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] compactions.Compactor(225): Compacting fc47320ccf9f4dd5997bc34aa0bb9c3e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1243, earliestPutTs=1731565131625 2024-11-14T06:18:53,732 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d062eac452eae155317450043cfef448#C#compaction#567 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T06:18:53,732 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/343f981b8a77416f8202b55cc6ae2525 is 50, key is test_row_0/C:col10/1731565131625/Put/seqid=0 2024-11-14T06:18:53,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742501_1677 (size=13323) 2024-11-14T06:18:53,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:18:53,790 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111426f42d3b6b624b15ab313d3091986d42_d062eac452eae155317450043cfef448 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111426f42d3b6b624b15ab313d3091986d42_d062eac452eae155317450043cfef448 2024-11-14T06:18:53,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/0ff3a438c41c43ae895174beed1f643e, store: [table=TestAcidGuarantees family=A region=d062eac452eae155317450043cfef448] 2024-11-14T06:18:53,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/0ff3a438c41c43ae895174beed1f643e is 175, key is test_row_0/A:col10/1731565132282/Put/seqid=0 2024-11-14T06:18:53,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742502_1678 (size=31255) 2024-11-14T06:18:54,120 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/4f1044113e4748439920bcb52f91351c as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/4f1044113e4748439920bcb52f91351c 2024-11-14T06:18:54,124 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d062eac452eae155317450043cfef448/A of d062eac452eae155317450043cfef448 into 4f1044113e4748439920bcb52f91351c(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:18:54,124 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d062eac452eae155317450043cfef448: 2024-11-14T06:18:54,124 INFO [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., storeName=d062eac452eae155317450043cfef448/A, priority=13, startTime=1731565133297; duration=0sec 2024-11-14T06:18:54,124 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:54,124 DEBUG [RS:0;ef93ef3a83c5:42283-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d062eac452eae155317450043cfef448:A 2024-11-14T06:18:54,138 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/343f981b8a77416f8202b55cc6ae2525 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/343f981b8a77416f8202b55cc6ae2525 2024-11-14T06:18:54,142 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d062eac452eae155317450043cfef448/C of d062eac452eae155317450043cfef448 into 343f981b8a77416f8202b55cc6ae2525(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T06:18:54,142 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d062eac452eae155317450043cfef448: 2024-11-14T06:18:54,142 INFO [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., storeName=d062eac452eae155317450043cfef448/C, priority=13, startTime=1731565133297; duration=0sec 2024-11-14T06:18:54,142 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T06:18:54,142 DEBUG [RS:0;ef93ef3a83c5:42283-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d062eac452eae155317450043cfef448:C 2024-11-14T06:18:54,195 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=1257, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/0ff3a438c41c43ae895174beed1f643e 2024-11-14T06:18:54,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/13ec72a8fc2449748a85522bed708261 is 50, key is test_row_0/B:col10/1731565132282/Put/seqid=0 2024-11-14T06:18:54,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742503_1679 (size=12301) 2024-11-14T06:18:54,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-14T06:18:54,604 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=1257 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/13ec72a8fc2449748a85522bed708261 2024-11-14T06:18:54,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/95e8735d49b0431380e4fab263bb7c6a is 50, key is test_row_0/C:col10/1731565132282/Put/seqid=0 2024-11-14T06:18:54,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742504_1680 (size=12301) 2024-11-14T06:18:55,013 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=1257 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/95e8735d49b0431380e4fab263bb7c6a 2024-11-14T06:18:55,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/0ff3a438c41c43ae895174beed1f643e as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/0ff3a438c41c43ae895174beed1f643e 2024-11-14T06:18:55,019 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/0ff3a438c41c43ae895174beed1f643e, entries=150, sequenceid=1257, filesize=30.5 K 2024-11-14T06:18:55,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/13ec72a8fc2449748a85522bed708261 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/13ec72a8fc2449748a85522bed708261 2024-11-14T06:18:55,023 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/13ec72a8fc2449748a85522bed708261, entries=150, sequenceid=1257, filesize=12.0 K 2024-11-14T06:18:55,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/95e8735d49b0431380e4fab263bb7c6a as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/95e8735d49b0431380e4fab263bb7c6a 2024-11-14T06:18:55,025 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/95e8735d49b0431380e4fab263bb7c6a, entries=150, sequenceid=1257, filesize=12.0 K 2024-11-14T06:18:55,026 INFO [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=33.54 KB/34350 for d062eac452eae155317450043cfef448 in 1646ms, sequenceid=1257, compaction requested=false 2024-11-14T06:18:55,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for d062eac452eae155317450043cfef448: 2024-11-14T06:18:55,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:55,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef93ef3a83c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-14T06:18:55,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-14T06:18:55,028 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-14T06:18:55,028 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7310 sec 2024-11-14T06:18:55,029 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 2.7420 sec 2024-11-14T06:18:56,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-14T06:18:56,447 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:18:56,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T06:18:56,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 225 2024-11-14T06:18:56,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 192 2024-11-14T06:18:56,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 260 2024-11-14T06:18:56,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 254 2024-11-14T06:18:56,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 234 2024-11-14T06:18:56,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T06:18:56,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2596 2024-11-14T06:18:56,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2532 2024-11-14T06:18:56,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2550 2024-11-14T06:18:56,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2586 2024-11-14T06:18:56,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2516 2024-11-14T06:18:56,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T06:18:56,448 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T06:18:56,448 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T06:18:56,448 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:56,448 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:56,448 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:18:56,448 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-14T06:18:56,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T06:18:56,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-14T06:18:56,451 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731565136451"}]},"ts":"1731565136451"} 2024-11-14T06:18:56,530 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T06:18:56,530 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T06:18:56,531 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T06:18:56,532 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d062eac452eae155317450043cfef448, UNASSIGN}] 2024-11-14T06:18:56,532 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d062eac452eae155317450043cfef448, UNASSIGN 2024-11-14T06:18:56,533 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=166 updating hbase:meta row=d062eac452eae155317450043cfef448, regionState=CLOSING, regionLocation=ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:56,534 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d062eac452eae155317450043cfef448, UNASSIGN because future has completed 2024-11-14T06:18:56,535 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T06:18:56,535 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; CloseRegionProcedure d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945}] 2024-11-14T06:18:56,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-14T06:18:56,687 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(122): Close d062eac452eae155317450043cfef448 2024-11-14T06:18:56,687 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T06:18:56,687 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1722): Closing d062eac452eae155317450043cfef448, disabling compactions & flushes 2024-11-14T06:18:56,687 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:56,688 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:56,688 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. after waiting 0 ms 2024-11-14T06:18:56,688 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:56,688 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(2902): Flushing d062eac452eae155317450043cfef448 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-14T06:18:56,688 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=A 2024-11-14T06:18:56,688 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:56,688 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=B 2024-11-14T06:18:56,688 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:56,688 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d062eac452eae155317450043cfef448, store=C 2024-11-14T06:18:56,688 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T06:18:56,692 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114c9f95ba565714349a33a2fae9f906e01_d062eac452eae155317450043cfef448 is 50, key is test_row_0/A:col10/1731565133447/Put/seqid=0 2024-11-14T06:18:56,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742505_1681 (size=9914) 2024-11-14T06:18:56,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-14T06:18:57,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-14T06:18:57,095 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T06:18:57,098 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114c9f95ba565714349a33a2fae9f906e01_d062eac452eae155317450043cfef448 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c9f95ba565714349a33a2fae9f906e01_d062eac452eae155317450043cfef448 2024-11-14T06:18:57,098 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/1ede2717098f4a7080519d234373cbed, store: [table=TestAcidGuarantees family=A region=d062eac452eae155317450043cfef448] 2024-11-14T06:18:57,099 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/1ede2717098f4a7080519d234373cbed is 175, key is test_row_0/A:col10/1731565133447/Put/seqid=0 2024-11-14T06:18:57,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742506_1682 (size=22561) 2024-11-14T06:18:57,502 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=1268, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/1ede2717098f4a7080519d234373cbed 2024-11-14T06:18:57,508 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/8079d001b6da407481fa350529a21d6c is 50, key is test_row_0/B:col10/1731565133447/Put/seqid=0 2024-11-14T06:18:57,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742507_1683 (size=9857) 2024-11-14T06:18:57,511 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=1268 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/8079d001b6da407481fa350529a21d6c 2024-11-14T06:18:57,516 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/bc52559923fc416eb1b5d5e0c8160806 is 50, key is test_row_0/C:col10/1731565133447/Put/seqid=0 2024-11-14T06:18:57,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742508_1684 (size=9857) 2024-11-14T06:18:57,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-14T06:18:57,921 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=1268 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/bc52559923fc416eb1b5d5e0c8160806 2024-11-14T06:18:57,925 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/A/1ede2717098f4a7080519d234373cbed as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/1ede2717098f4a7080519d234373cbed 2024-11-14T06:18:57,927 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/1ede2717098f4a7080519d234373cbed, entries=100, sequenceid=1268, filesize=22.0 K 2024-11-14T06:18:57,928 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/B/8079d001b6da407481fa350529a21d6c as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/8079d001b6da407481fa350529a21d6c 2024-11-14T06:18:57,930 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/8079d001b6da407481fa350529a21d6c, entries=100, sequenceid=1268, filesize=9.6 K 2024-11-14T06:18:57,930 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/.tmp/C/bc52559923fc416eb1b5d5e0c8160806 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/bc52559923fc416eb1b5d5e0c8160806 2024-11-14T06:18:57,933 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/bc52559923fc416eb1b5d5e0c8160806, entries=100, sequenceid=1268, filesize=9.6 K 2024-11-14T06:18:57,933 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for d062eac452eae155317450043cfef448 in 1245ms, sequenceid=1268, compaction requested=true 2024-11-14T06:18:57,934 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/5ad92034a9434cd38febcd8ce885cdef, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/d2d29244d19f4389aaf0a062f51f99e4, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/673cd1e90d224b348e6b7780ced448fa, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/c9c34a8f8f3e4a6192a6a07173e2df10, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/4d778e54da8c4967bc3d2ee1546d2145, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/eeed945c9aa543a585c59b924ee68eaa, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/0ad81e870c754eea95abcae13621ee9c, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/193507e26d9d44ec9e0fde2b26b73e92, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/47256b6c7b5c4a6495b64179663f3560, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/1565e4c4c54447f59813b4913e6e261a, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/aa0321ef1598490e989b079f14c4c0b7, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/cf7c6618f20e45969f905c013f13850d, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/143491bc728d460290cd1d1dc97b028e, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/8e7acf0dc14047c78dd1894ef528e31b, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/5bc89321357544138a130c60eede428c, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/ceb397c17c7a49e3963f0c9372a84187, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/b58735e053ee4b86a217e25250c7e1e2, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/47d62f1beabf46f9a5114842d263684e, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/848d7e4ec4574d5db96e77ef12145ff4, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/82936432b5474561973b079d21605b91, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/4993d807f91748ca9434a4faad1d349b, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/4f700b3a47ae4817a59b0265c5599483, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/32d38ab6ebe541ae8ce3f0c9a1e594a4, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/774faa2243864c2abeefc2f3a13f1187, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/6db3c5ee34294a179d13fa6e1aa9eaab, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/6fec64f49e0f4eb0aadd833de837bb53, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/6268ab67e7984700a19f1354e7fcedad, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/325127a356c34f01863d503240d9f964, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/26f6d3c5cab14d5f8209d83f3be8bea9, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/92573a53ca23415990acb308ce53470d] to archive 2024-11-14T06:18:57,934 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T06:18:57,936 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/5ad92034a9434cd38febcd8ce885cdef to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/5ad92034a9434cd38febcd8ce885cdef 2024-11-14T06:18:57,936 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/d2d29244d19f4389aaf0a062f51f99e4 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/d2d29244d19f4389aaf0a062f51f99e4 2024-11-14T06:18:57,937 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/673cd1e90d224b348e6b7780ced448fa to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/673cd1e90d224b348e6b7780ced448fa 2024-11-14T06:18:57,938 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/c9c34a8f8f3e4a6192a6a07173e2df10 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/c9c34a8f8f3e4a6192a6a07173e2df10 2024-11-14T06:18:57,938 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/4d778e54da8c4967bc3d2ee1546d2145 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/4d778e54da8c4967bc3d2ee1546d2145 2024-11-14T06:18:57,939 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/eeed945c9aa543a585c59b924ee68eaa to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/eeed945c9aa543a585c59b924ee68eaa 2024-11-14T06:18:57,940 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/0ad81e870c754eea95abcae13621ee9c to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/0ad81e870c754eea95abcae13621ee9c 2024-11-14T06:18:57,940 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/193507e26d9d44ec9e0fde2b26b73e92 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/193507e26d9d44ec9e0fde2b26b73e92 2024-11-14T06:18:57,941 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/47256b6c7b5c4a6495b64179663f3560 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/47256b6c7b5c4a6495b64179663f3560 2024-11-14T06:18:57,942 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/1565e4c4c54447f59813b4913e6e261a to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/1565e4c4c54447f59813b4913e6e261a 2024-11-14T06:18:57,942 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/aa0321ef1598490e989b079f14c4c0b7 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/aa0321ef1598490e989b079f14c4c0b7 2024-11-14T06:18:57,943 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/cf7c6618f20e45969f905c013f13850d to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/cf7c6618f20e45969f905c013f13850d 2024-11-14T06:18:57,944 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/143491bc728d460290cd1d1dc97b028e to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/143491bc728d460290cd1d1dc97b028e 2024-11-14T06:18:57,944 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/8e7acf0dc14047c78dd1894ef528e31b to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/8e7acf0dc14047c78dd1894ef528e31b 2024-11-14T06:18:57,945 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/5bc89321357544138a130c60eede428c to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/5bc89321357544138a130c60eede428c 2024-11-14T06:18:57,946 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/ceb397c17c7a49e3963f0c9372a84187 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/ceb397c17c7a49e3963f0c9372a84187 2024-11-14T06:18:57,946 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/b58735e053ee4b86a217e25250c7e1e2 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/b58735e053ee4b86a217e25250c7e1e2 2024-11-14T06:18:57,947 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/47d62f1beabf46f9a5114842d263684e to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/47d62f1beabf46f9a5114842d263684e 2024-11-14T06:18:57,948 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/848d7e4ec4574d5db96e77ef12145ff4 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/848d7e4ec4574d5db96e77ef12145ff4 2024-11-14T06:18:57,948 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/82936432b5474561973b079d21605b91 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/82936432b5474561973b079d21605b91 2024-11-14T06:18:57,949 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/4993d807f91748ca9434a4faad1d349b to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/4993d807f91748ca9434a4faad1d349b 2024-11-14T06:18:57,950 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/4f700b3a47ae4817a59b0265c5599483 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/4f700b3a47ae4817a59b0265c5599483 2024-11-14T06:18:57,950 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/32d38ab6ebe541ae8ce3f0c9a1e594a4 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/32d38ab6ebe541ae8ce3f0c9a1e594a4 2024-11-14T06:18:57,951 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/774faa2243864c2abeefc2f3a13f1187 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/774faa2243864c2abeefc2f3a13f1187 2024-11-14T06:18:57,951 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/6db3c5ee34294a179d13fa6e1aa9eaab to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/6db3c5ee34294a179d13fa6e1aa9eaab 2024-11-14T06:18:57,952 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/6fec64f49e0f4eb0aadd833de837bb53 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/6fec64f49e0f4eb0aadd833de837bb53 2024-11-14T06:18:57,953 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/6268ab67e7984700a19f1354e7fcedad to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/6268ab67e7984700a19f1354e7fcedad 2024-11-14T06:18:57,954 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/325127a356c34f01863d503240d9f964 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/325127a356c34f01863d503240d9f964 2024-11-14T06:18:57,955 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/26f6d3c5cab14d5f8209d83f3be8bea9 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/26f6d3c5cab14d5f8209d83f3be8bea9 2024-11-14T06:18:57,956 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/92573a53ca23415990acb308ce53470d to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/92573a53ca23415990acb308ce53470d 2024-11-14T06:18:57,957 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/07a79771b9594786ade9fce2aa331319, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/027ad5a6fc3249999a0f861f4fa1c034, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/a905ce88736c4c518b72d5adf4544e8f, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/437b5a7ac29d426aab7cfcc2d30329c4, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/7d986465ea094ff19142a57d7b6b8198, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/938a72eaff8848d680dd98c120f2c359, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/b3a4b1a79a79406791ca25d5c25781c2, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/7d8bc2b45817426f9a8d4bff092c1e6b, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/410ce9360959481cab7f0dba9133cf9f, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/67d4a97def144407971d9e69d5302ed5, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/ff9d569439764f9d8f2302f69f1310dd, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/06ba1b23427942f4a7dede27626d9e1b, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/c459366c1ccc42f8921d54e41346480a, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/e450e0ca67304e95b7df979cf7ad9a7b, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/44c6f599aa044d87a7f7559fc4aa0765, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/d1afe333131d46f1bffc4ba3931751c5, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/9c4070c8c814480db653fa73ccc356da, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/d1a3ecb5318841d1888bdf3c0a2853ba, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/9f0bd28752614a0e8247e28e77180b7e, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/6922ec1be880493abf4e32a70992895e, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/d9f574421bc64fde81f93c056385a51c, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/f45f4d0293f64208b835f3fd01c82e18, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/e77d24738e754cf1b50d280dca0d2f01, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/38b38e681147441fbec665fdcea2e121, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/ca0289ba90b64730b3e8cb239fde3598, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/e8a814fb3dc44a9881dda7f24ba9df8c, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/b080ce8739ac479d8e5cd66ca6be61df, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/2ab5b879d2324e3d8962dbd2cd154c7b, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/e81fb802c7844d00b1c81e2f2b333c7b, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/cf2326998c194b179a5cacbc75671a2d] to archive 2024-11-14T06:18:57,958 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T06:18:57,959 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/07a79771b9594786ade9fce2aa331319 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/07a79771b9594786ade9fce2aa331319 2024-11-14T06:18:57,960 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/027ad5a6fc3249999a0f861f4fa1c034 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/027ad5a6fc3249999a0f861f4fa1c034 2024-11-14T06:18:57,961 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/a905ce88736c4c518b72d5adf4544e8f to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/a905ce88736c4c518b72d5adf4544e8f 2024-11-14T06:18:57,962 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/437b5a7ac29d426aab7cfcc2d30329c4 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/437b5a7ac29d426aab7cfcc2d30329c4 2024-11-14T06:18:57,963 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/7d986465ea094ff19142a57d7b6b8198 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/7d986465ea094ff19142a57d7b6b8198 2024-11-14T06:18:57,964 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/938a72eaff8848d680dd98c120f2c359 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/938a72eaff8848d680dd98c120f2c359 2024-11-14T06:18:57,965 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/b3a4b1a79a79406791ca25d5c25781c2 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/b3a4b1a79a79406791ca25d5c25781c2 2024-11-14T06:18:57,966 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/7d8bc2b45817426f9a8d4bff092c1e6b to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/7d8bc2b45817426f9a8d4bff092c1e6b 2024-11-14T06:18:57,967 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/410ce9360959481cab7f0dba9133cf9f to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/410ce9360959481cab7f0dba9133cf9f 2024-11-14T06:18:57,968 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/67d4a97def144407971d9e69d5302ed5 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/67d4a97def144407971d9e69d5302ed5 2024-11-14T06:18:57,968 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/ff9d569439764f9d8f2302f69f1310dd to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/ff9d569439764f9d8f2302f69f1310dd 2024-11-14T06:18:57,969 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/06ba1b23427942f4a7dede27626d9e1b to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/06ba1b23427942f4a7dede27626d9e1b 2024-11-14T06:18:57,970 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/c459366c1ccc42f8921d54e41346480a to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/c459366c1ccc42f8921d54e41346480a 2024-11-14T06:18:57,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/e450e0ca67304e95b7df979cf7ad9a7b to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/e450e0ca67304e95b7df979cf7ad9a7b 2024-11-14T06:18:57,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/44c6f599aa044d87a7f7559fc4aa0765 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/44c6f599aa044d87a7f7559fc4aa0765 2024-11-14T06:18:57,972 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/d1afe333131d46f1bffc4ba3931751c5 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/d1afe333131d46f1bffc4ba3931751c5 2024-11-14T06:18:57,973 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/9c4070c8c814480db653fa73ccc356da to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/9c4070c8c814480db653fa73ccc356da 2024-11-14T06:18:57,973 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/d1a3ecb5318841d1888bdf3c0a2853ba to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/d1a3ecb5318841d1888bdf3c0a2853ba 2024-11-14T06:18:57,974 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/9f0bd28752614a0e8247e28e77180b7e to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/9f0bd28752614a0e8247e28e77180b7e 2024-11-14T06:18:57,975 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/6922ec1be880493abf4e32a70992895e to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/6922ec1be880493abf4e32a70992895e 2024-11-14T06:18:57,975 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/d9f574421bc64fde81f93c056385a51c to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/d9f574421bc64fde81f93c056385a51c 2024-11-14T06:18:57,976 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/f45f4d0293f64208b835f3fd01c82e18 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/f45f4d0293f64208b835f3fd01c82e18 2024-11-14T06:18:57,977 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/e77d24738e754cf1b50d280dca0d2f01 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/e77d24738e754cf1b50d280dca0d2f01 2024-11-14T06:18:57,977 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/38b38e681147441fbec665fdcea2e121 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/38b38e681147441fbec665fdcea2e121 2024-11-14T06:18:57,978 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/ca0289ba90b64730b3e8cb239fde3598 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/ca0289ba90b64730b3e8cb239fde3598 2024-11-14T06:18:57,979 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/e8a814fb3dc44a9881dda7f24ba9df8c to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/e8a814fb3dc44a9881dda7f24ba9df8c 2024-11-14T06:18:57,979 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/b080ce8739ac479d8e5cd66ca6be61df to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/b080ce8739ac479d8e5cd66ca6be61df 2024-11-14T06:18:57,980 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/2ab5b879d2324e3d8962dbd2cd154c7b to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/2ab5b879d2324e3d8962dbd2cd154c7b 2024-11-14T06:18:57,981 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/e81fb802c7844d00b1c81e2f2b333c7b to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/e81fb802c7844d00b1c81e2f2b333c7b 2024-11-14T06:18:57,982 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/cf2326998c194b179a5cacbc75671a2d to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/cf2326998c194b179a5cacbc75671a2d 2024-11-14T06:18:57,983 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/97f94f010c7049d692542d3d9261971a, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/d6340760470b4080883adfedf44732b5, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/fbbae768cd9a49f3a6d2a88189c1c206, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/2d57cc5dc0264ec19458ce5a53e5c191, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/6c5fd27fa1e3463e8bc8e4d96dd09edb, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/9c81ad76ee1c4f15a191540de38bea88, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/21bded997e4b436cb05d691f7876e983, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/0c2dc43b929a4bf088df1ca94482af1d, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/d703950c626f47a2952f0220096837f4, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/889599a6a8f94ca499876f46c3116e64, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/7cf09b05e01743eb92024d5f03c8faeb, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/5e7b5dd5dcba43deab711e1bbc99d3dc, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/00991d7cbaa14366b6b5bd73102565ba, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/2be3ddd958854833b55ae6d59f42130b, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/db9b9f15dd2043bfae34c215315c24f8, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/e0011cb0fa4b4ad4b50d0935fac8f92a, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/f304c828e52c40faa6a949caca08c344, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/c61547414e684c27b4c65366a7c18fe7, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/36efa621b213439c9bbd76d6627c1f17, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/d52c18fbf82141c08b05496681bc0b89, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/1fd2d8f9c0a149b29db5148a0d181b57, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/a90a94b66f644432a37f6498f4d86a51, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/715256f087344d75bff598272ed37d1f, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/a2c152b0203e415c8842a73ebccc44fa, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/35c8b3fcd8654aa6aaac74c0a5e5112a, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/7e2dc93eaca74776bfed09ef83889686, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/0b24e0b4f4b847dd8fad2a64ab419831, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/c4442dc47652491489eb392d6e6e4456, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/d73e0d5027cf4aab84d6e4f1c2ce133d, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/fc47320ccf9f4dd5997bc34aa0bb9c3e] to archive 2024-11-14T06:18:57,984 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T06:18:57,985 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/97f94f010c7049d692542d3d9261971a to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/97f94f010c7049d692542d3d9261971a 2024-11-14T06:18:57,986 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/d6340760470b4080883adfedf44732b5 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/d6340760470b4080883adfedf44732b5 2024-11-14T06:18:57,986 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/fbbae768cd9a49f3a6d2a88189c1c206 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/fbbae768cd9a49f3a6d2a88189c1c206 2024-11-14T06:18:57,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/2d57cc5dc0264ec19458ce5a53e5c191 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/2d57cc5dc0264ec19458ce5a53e5c191 2024-11-14T06:18:57,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/6c5fd27fa1e3463e8bc8e4d96dd09edb to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/6c5fd27fa1e3463e8bc8e4d96dd09edb 2024-11-14T06:18:57,988 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/9c81ad76ee1c4f15a191540de38bea88 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/9c81ad76ee1c4f15a191540de38bea88 2024-11-14T06:18:57,989 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/21bded997e4b436cb05d691f7876e983 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/21bded997e4b436cb05d691f7876e983 2024-11-14T06:18:57,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/0c2dc43b929a4bf088df1ca94482af1d to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/0c2dc43b929a4bf088df1ca94482af1d 2024-11-14T06:18:57,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/d703950c626f47a2952f0220096837f4 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/d703950c626f47a2952f0220096837f4 2024-11-14T06:18:57,991 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/889599a6a8f94ca499876f46c3116e64 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/889599a6a8f94ca499876f46c3116e64 2024-11-14T06:18:57,992 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/7cf09b05e01743eb92024d5f03c8faeb to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/7cf09b05e01743eb92024d5f03c8faeb 2024-11-14T06:18:57,993 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/5e7b5dd5dcba43deab711e1bbc99d3dc to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/5e7b5dd5dcba43deab711e1bbc99d3dc 2024-11-14T06:18:57,993 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/00991d7cbaa14366b6b5bd73102565ba to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/00991d7cbaa14366b6b5bd73102565ba 2024-11-14T06:18:57,994 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/2be3ddd958854833b55ae6d59f42130b to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/2be3ddd958854833b55ae6d59f42130b 2024-11-14T06:18:57,995 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/db9b9f15dd2043bfae34c215315c24f8 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/db9b9f15dd2043bfae34c215315c24f8 2024-11-14T06:18:57,996 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/e0011cb0fa4b4ad4b50d0935fac8f92a to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/e0011cb0fa4b4ad4b50d0935fac8f92a 2024-11-14T06:18:57,996 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/f304c828e52c40faa6a949caca08c344 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/f304c828e52c40faa6a949caca08c344 2024-11-14T06:18:57,997 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/c61547414e684c27b4c65366a7c18fe7 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/c61547414e684c27b4c65366a7c18fe7 2024-11-14T06:18:57,998 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/36efa621b213439c9bbd76d6627c1f17 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/36efa621b213439c9bbd76d6627c1f17 2024-11-14T06:18:57,999 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/d52c18fbf82141c08b05496681bc0b89 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/d52c18fbf82141c08b05496681bc0b89 2024-11-14T06:18:57,999 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/1fd2d8f9c0a149b29db5148a0d181b57 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/1fd2d8f9c0a149b29db5148a0d181b57 2024-11-14T06:18:58,000 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/a90a94b66f644432a37f6498f4d86a51 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/a90a94b66f644432a37f6498f4d86a51 2024-11-14T06:18:58,002 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/715256f087344d75bff598272ed37d1f to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/715256f087344d75bff598272ed37d1f 2024-11-14T06:18:58,003 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/a2c152b0203e415c8842a73ebccc44fa to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/a2c152b0203e415c8842a73ebccc44fa 2024-11-14T06:18:58,003 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/35c8b3fcd8654aa6aaac74c0a5e5112a to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/35c8b3fcd8654aa6aaac74c0a5e5112a 2024-11-14T06:18:58,004 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/7e2dc93eaca74776bfed09ef83889686 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/7e2dc93eaca74776bfed09ef83889686 2024-11-14T06:18:58,005 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/0b24e0b4f4b847dd8fad2a64ab419831 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/0b24e0b4f4b847dd8fad2a64ab419831 2024-11-14T06:18:58,006 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/c4442dc47652491489eb392d6e6e4456 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/c4442dc47652491489eb392d6e6e4456 2024-11-14T06:18:58,007 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/d73e0d5027cf4aab84d6e4f1c2ce133d to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/d73e0d5027cf4aab84d6e4f1c2ce133d 2024-11-14T06:18:58,007 DEBUG [StoreCloser-TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/fc47320ccf9f4dd5997bc34aa0bb9c3e to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/fc47320ccf9f4dd5997bc34aa0bb9c3e 2024-11-14T06:18:58,011 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/recovered.edits/1271.seqid, newMaxSeqId=1271, maxSeqId=4 2024-11-14T06:18:58,012 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. 2024-11-14T06:18:58,012 DEBUG [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1676): Region close journal for d062eac452eae155317450043cfef448: Waiting for close lock at 1731565136687Running coprocessor pre-close hooks at 1731565136687Disabling compacts and flushes for region at 1731565136687Disabling writes for close at 1731565136688 (+1 ms)Obtaining lock to block concurrent updates at 1731565136688Preparing flush snapshotting stores in d062eac452eae155317450043cfef448 at 1731565136688Finished memstore snapshotting TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731565136688Flushing stores of TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448. at 1731565136688Flushing d062eac452eae155317450043cfef448/A: creating writer at 1731565136688Flushing mob file d062eac452eae155317450043cfef448/A: appending metadata at 1731565136692 (+4 ms)Flushing mob file d062eac452eae155317450043cfef448/A: closing flushed file at 1731565136692Flushing d062eac452eae155317450043cfef448/A: appending metadata at 1731565137098 (+406 ms)Flushing d062eac452eae155317450043cfef448/A: closing flushed file at 1731565137099 (+1 ms)Flushing d062eac452eae155317450043cfef448/B: creating writer at 1731565137506 (+407 ms)Flushing d062eac452eae155317450043cfef448/B: appending metadata at 1731565137508 (+2 ms)Flushing d062eac452eae155317450043cfef448/B: closing flushed file at 1731565137508Flushing d062eac452eae155317450043cfef448/C: creating writer at 1731565137514 (+6 ms)Flushing d062eac452eae155317450043cfef448/C: appending metadata at 1731565137516 (+2 ms)Flushing d062eac452eae155317450043cfef448/C: closing flushed file at 1731565137516Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a82f045: reopening flushed file at 1731565137924 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6befa902: reopening flushed file at 1731565137927 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1046c24d: reopening flushed file at 1731565137930 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for d062eac452eae155317450043cfef448 in 1245ms, sequenceid=1268, compaction requested=true at 1731565137933 (+3 ms)Writing region close event to WAL at 1731565138009 (+76 ms)Running coprocessor post-close hooks at 1731565138012 (+3 ms)Closed at 1731565138012 2024-11-14T06:18:58,014 INFO [RS_CLOSE_REGION-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(157): Closed d062eac452eae155317450043cfef448 2024-11-14T06:18:58,014 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=166 updating hbase:meta row=d062eac452eae155317450043cfef448, regionState=CLOSED 2024-11-14T06:18:58,015 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=167, ppid=166, state=RUNNABLE, hasLock=false; CloseRegionProcedure d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 because future has completed 2024-11-14T06:18:58,018 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-14T06:18:58,018 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; CloseRegionProcedure d062eac452eae155317450043cfef448, server=ef93ef3a83c5,42283,1731564965945 in 1.4810 sec 2024-11-14T06:18:58,019 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=166, resume processing ppid=165 2024-11-14T06:18:58,019 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, ppid=165, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d062eac452eae155317450043cfef448, UNASSIGN in 1.4860 sec 2024-11-14T06:18:58,021 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-14T06:18:58,021 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4890 sec 2024-11-14T06:18:58,022 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731565138022"}]},"ts":"1731565138022"} 2024-11-14T06:18:58,023 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T06:18:58,023 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T06:18:58,025 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5760 sec 2024-11-14T06:18:58,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-14T06:18:58,598 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:18:58,598 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-14T06:18:58,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T06:18:58,600 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=168, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T06:18:58,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-14T06:18:58,601 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=168, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T06:18:58,604 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448 2024-11-14T06:18:58,606 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A, FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B, FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C, FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/recovered.edits] 2024-11-14T06:18:58,609 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/0ff3a438c41c43ae895174beed1f643e to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/0ff3a438c41c43ae895174beed1f643e 2024-11-14T06:18:58,611 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/1ede2717098f4a7080519d234373cbed to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/1ede2717098f4a7080519d234373cbed 2024-11-14T06:18:58,613 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/4f1044113e4748439920bcb52f91351c to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/A/4f1044113e4748439920bcb52f91351c 2024-11-14T06:18:58,616 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/13ec72a8fc2449748a85522bed708261 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/13ec72a8fc2449748a85522bed708261 2024-11-14T06:18:58,617 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/8079d001b6da407481fa350529a21d6c to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/8079d001b6da407481fa350529a21d6c 2024-11-14T06:18:58,618 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/d4fce2333310452a94c5ca14e9a9b155 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/B/d4fce2333310452a94c5ca14e9a9b155 2024-11-14T06:18:58,621 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/343f981b8a77416f8202b55cc6ae2525 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/343f981b8a77416f8202b55cc6ae2525 2024-11-14T06:18:58,622 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/95e8735d49b0431380e4fab263bb7c6a to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/95e8735d49b0431380e4fab263bb7c6a 2024-11-14T06:18:58,624 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/bc52559923fc416eb1b5d5e0c8160806 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/C/bc52559923fc416eb1b5d5e0c8160806 2024-11-14T06:18:58,631 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/recovered.edits/1271.seqid to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448/recovered.edits/1271.seqid 2024-11-14T06:18:58,631 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/default/TestAcidGuarantees/d062eac452eae155317450043cfef448 2024-11-14T06:18:58,632 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T06:18:58,632 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T06:18:58,633 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-14T06:18:58,637 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111401b60a1cc7f34bb28043d9171ca6fbfe_d062eac452eae155317450043cfef448 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111401b60a1cc7f34bb28043d9171ca6fbfe_d062eac452eae155317450043cfef448 2024-11-14T06:18:58,638 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411140efc4da5949f4e61a34936943f1caa28_d062eac452eae155317450043cfef448 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411140efc4da5949f4e61a34936943f1caa28_d062eac452eae155317450043cfef448 2024-11-14T06:18:58,639 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411140fe235d14d4c48c8bf58488625b45796_d062eac452eae155317450043cfef448 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411140fe235d14d4c48c8bf58488625b45796_d062eac452eae155317450043cfef448 2024-11-14T06:18:58,640 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111424c1b86ea7b4433bbf309fd8a787e9d6_d062eac452eae155317450043cfef448 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111424c1b86ea7b4433bbf309fd8a787e9d6_d062eac452eae155317450043cfef448 2024-11-14T06:18:58,642 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111426f42d3b6b624b15ab313d3091986d42_d062eac452eae155317450043cfef448 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111426f42d3b6b624b15ab313d3091986d42_d062eac452eae155317450043cfef448 2024-11-14T06:18:58,643 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411142b160948ea044e13acde8c2372ade99a_d062eac452eae155317450043cfef448 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411142b160948ea044e13acde8c2372ade99a_d062eac452eae155317450043cfef448 2024-11-14T06:18:58,644 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111435116cc3b5284af282c8166b0eb9af57_d062eac452eae155317450043cfef448 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111435116cc3b5284af282c8166b0eb9af57_d062eac452eae155317450043cfef448 2024-11-14T06:18:58,646 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143668c70ea87b4a5e952ce32e2f77d2f5_d062eac452eae155317450043cfef448 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143668c70ea87b4a5e952ce32e2f77d2f5_d062eac452eae155317450043cfef448 2024-11-14T06:18:58,647 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111449307b894d594f4080b4ea4e968e117e_d062eac452eae155317450043cfef448 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111449307b894d594f4080b4ea4e968e117e_d062eac452eae155317450043cfef448 2024-11-14T06:18:58,648 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145b8d36e7aa774a979d0e1c641677f689_d062eac452eae155317450043cfef448 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145b8d36e7aa774a979d0e1c641677f689_d062eac452eae155317450043cfef448 2024-11-14T06:18:58,649 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411146158c39b20b54559a523e1923c6ee94c_d062eac452eae155317450043cfef448 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411146158c39b20b54559a523e1923c6ee94c_d062eac452eae155317450043cfef448 2024-11-14T06:18:58,649 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411146d860e78b01e4679878212778c20d652_d062eac452eae155317450043cfef448 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411146d860e78b01e4679878212778c20d652_d062eac452eae155317450043cfef448 2024-11-14T06:18:58,650 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114aba46c063f8843b0b02e285da7f52b18_d062eac452eae155317450043cfef448 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114aba46c063f8843b0b02e285da7f52b18_d062eac452eae155317450043cfef448 2024-11-14T06:18:58,651 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ac75ebbf3d7345ddb90c7b6af12c44a9_d062eac452eae155317450043cfef448 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ac75ebbf3d7345ddb90c7b6af12c44a9_d062eac452eae155317450043cfef448 2024-11-14T06:18:58,652 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c358b3fd1b5e42608cc2e99e00ecdace_d062eac452eae155317450043cfef448 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c358b3fd1b5e42608cc2e99e00ecdace_d062eac452eae155317450043cfef448 2024-11-14T06:18:58,653 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c9f95ba565714349a33a2fae9f906e01_d062eac452eae155317450043cfef448 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c9f95ba565714349a33a2fae9f906e01_d062eac452eae155317450043cfef448 2024-11-14T06:18:58,654 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114caf8de77b8f84fd2b1a41bdeab2050da_d062eac452eae155317450043cfef448 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114caf8de77b8f84fd2b1a41bdeab2050da_d062eac452eae155317450043cfef448 2024-11-14T06:18:58,655 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114cd971e1750f848c286f3acef47be3126_d062eac452eae155317450043cfef448 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114cd971e1750f848c286f3acef47be3126_d062eac452eae155317450043cfef448 2024-11-14T06:18:58,656 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d311f2a85c9d4e969b3f2ca0f0c6ef05_d062eac452eae155317450043cfef448 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d311f2a85c9d4e969b3f2ca0f0c6ef05_d062eac452eae155317450043cfef448 2024-11-14T06:18:58,657 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d4e4e01a8fd744e1af77c008aeb33b01_d062eac452eae155317450043cfef448 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d4e4e01a8fd744e1af77c008aeb33b01_d062eac452eae155317450043cfef448 2024-11-14T06:18:58,660 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d64b3fcd48934ff8a324d48fd50a97df_d062eac452eae155317450043cfef448 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d64b3fcd48934ff8a324d48fd50a97df_d062eac452eae155317450043cfef448 2024-11-14T06:18:58,663 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f22dd01b0d374ddca8b1219678cc2d0b_d062eac452eae155317450043cfef448 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f22dd01b0d374ddca8b1219678cc2d0b_d062eac452eae155317450043cfef448 2024-11-14T06:18:58,664 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f4914fafc4ab4f739d6f5a74e77d8b78_d062eac452eae155317450043cfef448 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f4914fafc4ab4f739d6f5a74e77d8b78_d062eac452eae155317450043cfef448 2024-11-14T06:18:58,665 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114face7b18fe374530b71942f689afe366_d062eac452eae155317450043cfef448 to hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114face7b18fe374530b71942f689afe366_d062eac452eae155317450043cfef448 2024-11-14T06:18:58,665 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T06:18:58,667 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=168, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T06:18:58,670 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T06:18:58,672 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T06:18:58,674 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=168, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T06:18:58,674 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T06:18:58,674 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731565138674"}]},"ts":"9223372036854775807"} 2024-11-14T06:18:58,676 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T06:18:58,676 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => d062eac452eae155317450043cfef448, NAME => 'TestAcidGuarantees,,1731565110054.d062eac452eae155317450043cfef448.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T06:18:58,676 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T06:18:58,677 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731565138676"}]},"ts":"9223372036854775807"} 2024-11-14T06:18:58,679 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T06:18:58,680 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=168, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T06:18:58,681 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 82 msec 2024-11-14T06:18:58,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-14T06:18:58,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T06:18:58,708 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T06:18:58,722 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=250 (was 250), OpenFileDescriptor=449 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=697 (was 696) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1653 (was 312) - AvailableMemoryMB LEAK? - 2024-11-14T06:18:58,722 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-14T06:18:58,722 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T06:18:58,722 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T06:18:58,722 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:58,722 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T06:18:58,722 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:58,722 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-14T06:18:58,722 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=958468940, stopped=false 2024-11-14T06:18:58,723 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=ef93ef3a83c5,36045,1731564965170 2024-11-14T06:18:58,777 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36045-0x10137fc85b70000, quorum=127.0.0.1:61013, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T06:18:58,777 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42283-0x10137fc85b70001, quorum=127.0.0.1:61013, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T06:18:58,777 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42283-0x10137fc85b70001, quorum=127.0.0.1:61013, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T06:18:58,777 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T06:18:58,777 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36045-0x10137fc85b70000, quorum=127.0.0.1:61013, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T06:18:58,778 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T06:18:58,778 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T06:18:58,778 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:58,778 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36045-0x10137fc85b70000, quorum=127.0.0.1:61013, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T06:18:58,779 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42283-0x10137fc85b70001, quorum=127.0.0.1:61013, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T06:18:58,779 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ef93ef3a83c5,42283,1731564965945' ***** 2024-11-14T06:18:58,779 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-14T06:18:58,780 INFO [RS:0;ef93ef3a83c5:42283 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-14T06:18:58,780 INFO [RS:0;ef93ef3a83c5:42283 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-14T06:18:58,780 INFO [RS:0;ef93ef3a83c5:42283 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-14T06:18:58,780 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-14T06:18:58,780 INFO [RS:0;ef93ef3a83c5:42283 {}] regionserver.HRegionServer(959): stopping server ef93ef3a83c5,42283,1731564965945 2024-11-14T06:18:58,780 INFO [RS:0;ef93ef3a83c5:42283 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T06:18:58,780 INFO [RS:0;ef93ef3a83c5:42283 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;ef93ef3a83c5:42283. 2024-11-14T06:18:58,780 DEBUG [RS:0;ef93ef3a83c5:42283 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T06:18:58,780 DEBUG [RS:0;ef93ef3a83c5:42283 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:18:58,780 INFO [RS:0;ef93ef3a83c5:42283 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-14T06:18:58,781 INFO [RS:0;ef93ef3a83c5:42283 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-14T06:18:58,781 INFO [RS:0;ef93ef3a83c5:42283 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-14T06:18:58,781 INFO [RS:0;ef93ef3a83c5:42283 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-14T06:18:58,781 INFO [RS:0;ef93ef3a83c5:42283 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-14T06:18:58,781 DEBUG [RS:0;ef93ef3a83c5:42283 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-14T06:18:58,782 DEBUG [RS:0;ef93ef3a83c5:42283 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T06:18:58,782 DEBUG [RS_CLOSE_META-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T06:18:58,782 INFO [RS_CLOSE_META-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T06:18:58,782 DEBUG [RS_CLOSE_META-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T06:18:58,782 DEBUG [RS_CLOSE_META-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T06:18:58,782 DEBUG [RS_CLOSE_META-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T06:18:58,782 INFO [RS_CLOSE_META-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-14T06:18:58,809 DEBUG [RS_CLOSE_META-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/hbase/meta/1588230740/.tmp/info/6eea2e2cafb44353b4b8312950b2a05b is 95, key is TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e./info:/1731564996134/DeleteFamily/seqid=0 2024-11-14T06:18:58,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742509_1685 (size=6514) 2024-11-14T06:18:58,861 INFO [regionserver/ef93ef3a83c5:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T06:18:58,982 DEBUG [RS:0;ef93ef3a83c5:42283 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T06:18:59,138 INFO [regionserver/ef93ef3a83c5:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-14T06:18:59,138 INFO [regionserver/ef93ef3a83c5:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-14T06:18:59,182 DEBUG [RS:0;ef93ef3a83c5:42283 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T06:18:59,214 INFO [RS_CLOSE_META-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/hbase/meta/1588230740/.tmp/info/6eea2e2cafb44353b4b8312950b2a05b 2024-11-14T06:18:59,230 DEBUG [RS_CLOSE_META-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/hbase/meta/1588230740/.tmp/ns/48f09a5e7abb4644b568bb67c5ee16ac is 93, key is TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e./ns:/1731564996093/DeleteFamily/seqid=0 2024-11-14T06:18:59,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742510_1686 (size=5906) 2024-11-14T06:18:59,382 DEBUG [RS:0;ef93ef3a83c5:42283 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T06:18:59,583 DEBUG [RS:0;ef93ef3a83c5:42283 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T06:18:59,640 INFO [RS_CLOSE_META-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/hbase/meta/1588230740/.tmp/ns/48f09a5e7abb4644b568bb67c5ee16ac 2024-11-14T06:18:59,681 DEBUG [RS_CLOSE_META-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/hbase/meta/1588230740/.tmp/rep_barrier/8efec10f31634a20a2c8d28d4b0eae4c is 102, key is TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e./rep_barrier:/1731564996093/DeleteFamily/seqid=0 2024-11-14T06:18:59,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742511_1687 (size=6025) 2024-11-14T06:18:59,783 DEBUG [RS:0;ef93ef3a83c5:42283 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T06:18:59,983 DEBUG [RS:0;ef93ef3a83c5:42283 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T06:19:00,089 INFO [RS_CLOSE_META-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/hbase/meta/1588230740/.tmp/rep_barrier/8efec10f31634a20a2c8d28d4b0eae4c 2024-11-14T06:19:00,120 DEBUG [RS_CLOSE_META-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/hbase/meta/1588230740/.tmp/table/67231941ab2144349a31ddab8e4cf3ce is 96, key is TestAcidGuarantees,,1731564969487.d7851abd86d644f97c54179c322b1c1e./table:/1731564996093/DeleteFamily/seqid=0 2024-11-14T06:19:00,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742512_1688 (size=5936) 2024-11-14T06:19:00,183 DEBUG [RS:0;ef93ef3a83c5:42283 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T06:19:00,383 DEBUG [RS:0;ef93ef3a83c5:42283 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T06:19:00,526 INFO [RS_CLOSE_META-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/hbase/meta/1588230740/.tmp/table/67231941ab2144349a31ddab8e4cf3ce 2024-11-14T06:19:00,530 DEBUG [RS_CLOSE_META-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/hbase/meta/1588230740/.tmp/info/6eea2e2cafb44353b4b8312950b2a05b as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/hbase/meta/1588230740/info/6eea2e2cafb44353b4b8312950b2a05b 2024-11-14T06:19:00,535 INFO [RS_CLOSE_META-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/hbase/meta/1588230740/info/6eea2e2cafb44353b4b8312950b2a05b, entries=12, sequenceid=90, filesize=6.4 K 2024-11-14T06:19:00,536 DEBUG [RS_CLOSE_META-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/hbase/meta/1588230740/.tmp/ns/48f09a5e7abb4644b568bb67c5ee16ac as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/hbase/meta/1588230740/ns/48f09a5e7abb4644b568bb67c5ee16ac 2024-11-14T06:19:00,540 INFO [RS_CLOSE_META-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/hbase/meta/1588230740/ns/48f09a5e7abb4644b568bb67c5ee16ac, entries=8, sequenceid=90, filesize=5.8 K 2024-11-14T06:19:00,541 DEBUG [RS_CLOSE_META-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/hbase/meta/1588230740/.tmp/rep_barrier/8efec10f31634a20a2c8d28d4b0eae4c as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/hbase/meta/1588230740/rep_barrier/8efec10f31634a20a2c8d28d4b0eae4c 2024-11-14T06:19:00,545 INFO [RS_CLOSE_META-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/hbase/meta/1588230740/rep_barrier/8efec10f31634a20a2c8d28d4b0eae4c, entries=6, sequenceid=90, filesize=5.9 K 2024-11-14T06:19:00,546 DEBUG [RS_CLOSE_META-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/hbase/meta/1588230740/.tmp/table/67231941ab2144349a31ddab8e4cf3ce as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/hbase/meta/1588230740/table/67231941ab2144349a31ddab8e4cf3ce 2024-11-14T06:19:00,549 INFO [RS_CLOSE_META-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/hbase/meta/1588230740/table/67231941ab2144349a31ddab8e4cf3ce, entries=7, sequenceid=90, filesize=5.8 K 2024-11-14T06:19:00,550 INFO [RS_CLOSE_META-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1768ms, sequenceid=90, compaction requested=false 2024-11-14T06:19:00,573 DEBUG [RS_CLOSE_META-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-14T06:19:00,574 DEBUG [RS_CLOSE_META-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T06:19:00,574 INFO [RS_CLOSE_META-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T06:19:00,574 DEBUG [RS_CLOSE_META-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731565138781Running coprocessor pre-close hooks at 1731565138782 (+1 ms)Disabling compacts and flushes for region at 1731565138782Disabling writes for close at 1731565138782Obtaining lock to block concurrent updates at 1731565138782Preparing flush snapshotting stores in 1588230740 at 1731565138782Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731565138783 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731565138783Flushing 1588230740/info: creating writer at 1731565138784 (+1 ms)Flushing 1588230740/info: appending metadata at 1731565138809 (+25 ms)Flushing 1588230740/info: closing flushed file at 1731565138809Flushing 1588230740/ns: creating writer at 1731565139217 (+408 ms)Flushing 1588230740/ns: appending metadata at 1731565139230 (+13 ms)Flushing 1588230740/ns: closing flushed file at 1731565139230Flushing 1588230740/rep_barrier: creating writer at 1731565139659 (+429 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731565139681 (+22 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731565139681Flushing 1588230740/table: creating writer at 1731565140099 (+418 ms)Flushing 1588230740/table: appending metadata at 1731565140120 (+21 ms)Flushing 1588230740/table: closing flushed file at 1731565140120Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@14a74e97: reopening flushed file at 1731565140530 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@27e47b75: reopening flushed file at 1731565140536 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7b3b271a: reopening flushed file at 1731565140541 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@413c49b5: reopening flushed file at 1731565140545 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1768ms, sequenceid=90, compaction requested=false at 1731565140550 (+5 ms)Writing region close event to WAL at 1731565140567 (+17 ms)Running coprocessor post-close hooks at 1731565140574 (+7 ms)Closed at 1731565140574 2024-11-14T06:19:00,574 DEBUG [RS_CLOSE_META-regionserver/ef93ef3a83c5:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-14T06:19:00,584 INFO [RS:0;ef93ef3a83c5:42283 {}] regionserver.HRegionServer(976): stopping server ef93ef3a83c5,42283,1731564965945; all regions closed. 2024-11-14T06:19:00,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741834_1010 (size=25344) 2024-11-14T06:19:00,604 DEBUG [RS:0;ef93ef3a83c5:42283 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/oldWALs 2024-11-14T06:19:00,604 INFO [RS:0;ef93ef3a83c5:42283 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ef93ef3a83c5%2C42283%2C1731564965945.meta:.meta(num 1731564968897) 2024-11-14T06:19:00,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741833_1009 (size=23416872) 2024-11-14T06:19:00,608 DEBUG [RS:0;ef93ef3a83c5:42283 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/oldWALs 2024-11-14T06:19:00,608 INFO [RS:0;ef93ef3a83c5:42283 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ef93ef3a83c5%2C42283%2C1731564965945:(num 1731564968308) 2024-11-14T06:19:00,608 DEBUG [RS:0;ef93ef3a83c5:42283 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T06:19:00,608 INFO [RS:0;ef93ef3a83c5:42283 {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T06:19:00,608 INFO [RS:0;ef93ef3a83c5:42283 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T06:19:00,609 INFO [RS:0;ef93ef3a83c5:42283 {}] hbase.ChoreService(370): Chore service for: regionserver/ef93ef3a83c5:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-14T06:19:00,609 INFO [RS:0;ef93ef3a83c5:42283 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T06:19:00,609 INFO [regionserver/ef93ef3a83c5:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T06:19:00,610 INFO [RS:0;ef93ef3a83c5:42283 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:42283 2024-11-14T06:19:00,682 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42283-0x10137fc85b70001, quorum=127.0.0.1:61013, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ef93ef3a83c5,42283,1731564965945 2024-11-14T06:19:00,682 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36045-0x10137fc85b70000, quorum=127.0.0.1:61013, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T06:19:00,682 INFO [RS:0;ef93ef3a83c5:42283 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T06:19:00,745 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ef93ef3a83c5,42283,1731564965945] 2024-11-14T06:19:00,756 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ef93ef3a83c5,42283,1731564965945 already deleted, retry=false 2024-11-14T06:19:00,756 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; ef93ef3a83c5,42283,1731564965945 expired; onlineServers=0 2024-11-14T06:19:00,757 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'ef93ef3a83c5,36045,1731564965170' ***** 2024-11-14T06:19:00,757 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-14T06:19:00,757 INFO [M:0;ef93ef3a83c5:36045 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T06:19:00,757 INFO [M:0;ef93ef3a83c5:36045 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T06:19:00,757 DEBUG [M:0;ef93ef3a83c5:36045 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-14T06:19:00,757 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-14T06:19:00,757 DEBUG [M:0;ef93ef3a83c5:36045 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-14T06:19:00,757 DEBUG [master/ef93ef3a83c5:0:becomeActiveMaster-HFileCleaner.small.0-1731564968013 {}] cleaner.HFileCleaner(306): Exit Thread[master/ef93ef3a83c5:0:becomeActiveMaster-HFileCleaner.small.0-1731564968013,5,FailOnTimeoutGroup] 2024-11-14T06:19:00,757 DEBUG [master/ef93ef3a83c5:0:becomeActiveMaster-HFileCleaner.large.0-1731564968013 {}] cleaner.HFileCleaner(306): Exit Thread[master/ef93ef3a83c5:0:becomeActiveMaster-HFileCleaner.large.0-1731564968013,5,FailOnTimeoutGroup] 2024-11-14T06:19:00,757 INFO [M:0;ef93ef3a83c5:36045 {}] hbase.ChoreService(370): Chore service for: master/ef93ef3a83c5:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-14T06:19:00,757 INFO [M:0;ef93ef3a83c5:36045 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T06:19:00,758 DEBUG [M:0;ef93ef3a83c5:36045 {}] master.HMaster(1795): Stopping service threads 2024-11-14T06:19:00,758 INFO [M:0;ef93ef3a83c5:36045 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-14T06:19:00,758 INFO [M:0;ef93ef3a83c5:36045 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T06:19:00,758 ERROR [M:0;ef93ef3a83c5:36045 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1396431307) connection to localhost/127.0.0.1:35361 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:35361,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-14T06:19:00,759 INFO [M:0;ef93ef3a83c5:36045 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T06:19:00,759 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-14T06:19:00,766 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36045-0x10137fc85b70000, quorum=127.0.0.1:61013, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-14T06:19:00,767 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36045-0x10137fc85b70000, quorum=127.0.0.1:61013, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T06:19:00,767 DEBUG [M:0;ef93ef3a83c5:36045 {}] zookeeper.ZKUtil(347): master:36045-0x10137fc85b70000, quorum=127.0.0.1:61013, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-14T06:19:00,767 WARN [M:0;ef93ef3a83c5:36045 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-14T06:19:00,767 INFO [M:0;ef93ef3a83c5:36045 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/.lastflushedseqids 2024-11-14T06:19:00,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742513_1689 (size=72) 2024-11-14T06:19:00,846 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42283-0x10137fc85b70001, quorum=127.0.0.1:61013, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T06:19:00,846 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42283-0x10137fc85b70001, quorum=127.0.0.1:61013, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T06:19:00,846 INFO [RS:0;ef93ef3a83c5:42283 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T06:19:00,846 INFO [RS:0;ef93ef3a83c5:42283 {}] regionserver.HRegionServer(1031): Exiting; stopping=ef93ef3a83c5,42283,1731564965945; zookeeper connection closed. 2024-11-14T06:19:00,847 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@50e279fc {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@50e279fc 2024-11-14T06:19:00,847 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-14T06:19:01,177 INFO [M:0;ef93ef3a83c5:36045 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T06:19:01,178 INFO [M:0;ef93ef3a83c5:36045 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-14T06:19:01,178 DEBUG [M:0;ef93ef3a83c5:36045 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T06:19:01,178 INFO [M:0;ef93ef3a83c5:36045 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T06:19:01,178 DEBUG [M:0;ef93ef3a83c5:36045 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T06:19:01,178 DEBUG [M:0;ef93ef3a83c5:36045 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T06:19:01,178 DEBUG [M:0;ef93ef3a83c5:36045 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T06:19:01,178 INFO [M:0;ef93ef3a83c5:36045 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=796.98 KB heapSize=977.98 KB 2024-11-14T06:19:01,196 DEBUG [M:0;ef93ef3a83c5:36045 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2c0838f2975148178ba253d1fd80d8f7 is 82, key is hbase:meta,,1/info:regioninfo/1731564968987/Put/seqid=0 2024-11-14T06:19:01,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742514_1690 (size=5672) 2024-11-14T06:19:01,600 INFO [M:0;ef93ef3a83c5:36045 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2244 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2c0838f2975148178ba253d1fd80d8f7 2024-11-14T06:19:01,622 DEBUG [M:0;ef93ef3a83c5:36045 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e16a8fa696fc4815b1c6fc8811c76cec is 2481, key is \x00\x00\x00\x00\x00\x00\x00\x8F/proc:d/1731565113212/Put/seqid=0 2024-11-14T06:19:01,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742515_1691 (size=45067) 2024-11-14T06:19:02,026 INFO [M:0;ef93ef3a83c5:36045 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=796.42 KB at sequenceid=2244 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e16a8fa696fc4815b1c6fc8811c76cec 2024-11-14T06:19:02,030 INFO [M:0;ef93ef3a83c5:36045 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e16a8fa696fc4815b1c6fc8811c76cec 2024-11-14T06:19:02,045 DEBUG [M:0;ef93ef3a83c5:36045 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/cd322b814d56405ab422ce441ec2d2b8 is 69, key is ef93ef3a83c5,42283,1731564965945/rs:state/1731564968049/Put/seqid=0 2024-11-14T06:19:02,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073742516_1692 (size=5156) 2024-11-14T06:19:02,449 INFO [M:0;ef93ef3a83c5:36045 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2244 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/cd322b814d56405ab422ce441ec2d2b8 2024-11-14T06:19:02,452 DEBUG [M:0;ef93ef3a83c5:36045 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2c0838f2975148178ba253d1fd80d8f7 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2c0838f2975148178ba253d1fd80d8f7 2024-11-14T06:19:02,455 INFO [M:0;ef93ef3a83c5:36045 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2c0838f2975148178ba253d1fd80d8f7, entries=8, sequenceid=2244, filesize=5.5 K 2024-11-14T06:19:02,455 DEBUG [M:0;ef93ef3a83c5:36045 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e16a8fa696fc4815b1c6fc8811c76cec as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e16a8fa696fc4815b1c6fc8811c76cec 2024-11-14T06:19:02,458 INFO [M:0;ef93ef3a83c5:36045 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e16a8fa696fc4815b1c6fc8811c76cec 2024-11-14T06:19:02,458 INFO [M:0;ef93ef3a83c5:36045 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e16a8fa696fc4815b1c6fc8811c76cec, entries=168, sequenceid=2244, filesize=44.0 K 2024-11-14T06:19:02,458 DEBUG [M:0;ef93ef3a83c5:36045 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/cd322b814d56405ab422ce441ec2d2b8 as hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/cd322b814d56405ab422ce441ec2d2b8 2024-11-14T06:19:02,460 INFO [M:0;ef93ef3a83c5:36045 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c89b511a-4d8e-37b8-d74e-467c47fddb02/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/cd322b814d56405ab422ce441ec2d2b8, entries=1, sequenceid=2244, filesize=5.0 K 2024-11-14T06:19:02,461 INFO [M:0;ef93ef3a83c5:36045 {}] regionserver.HRegion(3140): Finished flush of dataSize ~796.98 KB/816106, heapSize ~977.69 KB/1001152, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1283ms, sequenceid=2244, compaction requested=false 2024-11-14T06:19:02,462 INFO [M:0;ef93ef3a83c5:36045 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T06:19:02,462 DEBUG [M:0;ef93ef3a83c5:36045 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731565141178Disabling compacts and flushes for region at 1731565141178Disabling writes for close at 1731565141178Obtaining lock to block concurrent updates at 1731565141178Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731565141178Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=816106, getHeapSize=1001392, getOffHeapSize=0, getCellsCount=2437 at 1731565141178Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731565141179 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731565141179Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731565141196 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731565141196Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731565141603 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731565141621 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731565141621Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731565142030 (+409 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731565142044 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731565142044Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4732db72: reopening flushed file at 1731565142451 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b1e472f: reopening flushed file at 1731565142455 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5fa40314: reopening flushed file at 1731565142458 (+3 ms)Finished flush of dataSize ~796.98 KB/816106, heapSize ~977.69 KB/1001152, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1283ms, sequenceid=2244, compaction requested=false at 1731565142461 (+3 ms)Writing region close event to WAL at 1731565142462 (+1 ms)Closed at 1731565142462 2024-11-14T06:19:02,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33461 is added to blk_1073741830_1006 (size=963157) 2024-11-14T06:19:02,465 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T06:19:02,465 INFO [M:0;ef93ef3a83c5:36045 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-14T06:19:02,465 INFO [M:0;ef93ef3a83c5:36045 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:36045 2024-11-14T06:19:02,465 INFO [M:0;ef93ef3a83c5:36045 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T06:19:02,698 INFO [M:0;ef93ef3a83c5:36045 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T06:19:02,698 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36045-0x10137fc85b70000, quorum=127.0.0.1:61013, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T06:19:02,698 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36045-0x10137fc85b70000, quorum=127.0.0.1:61013, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T06:19:02,708 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@58e0e813{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T06:19:02,711 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@716e8981{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T06:19:02,711 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T06:19:02,711 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@50fd64ad{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T06:19:02,711 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7d058786{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5ae4a4a-2751-c4f4-7a28-da4c7cf058aa/hadoop.log.dir/,STOPPED} 2024-11-14T06:19:02,715 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-14T06:19:02,715 WARN [BP-507516446-172.17.0.3-1731564960889 heartbeating to localhost/127.0.0.1:35361 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-14T06:19:02,715 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-14T06:19:02,715 WARN [BP-507516446-172.17.0.3-1731564960889 heartbeating to localhost/127.0.0.1:35361 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-507516446-172.17.0.3-1731564960889 (Datanode Uuid 93aa84dc-c036-4732-ac22-9c38de32e706) service to localhost/127.0.0.1:35361 2024-11-14T06:19:02,718 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5ae4a4a-2751-c4f4-7a28-da4c7cf058aa/cluster_7988e05c-75f7-7e7d-81fe-ae6ab708e25a/data/data1/current/BP-507516446-172.17.0.3-1731564960889 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T06:19:02,718 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5ae4a4a-2751-c4f4-7a28-da4c7cf058aa/cluster_7988e05c-75f7-7e7d-81fe-ae6ab708e25a/data/data2/current/BP-507516446-172.17.0.3-1731564960889 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T06:19:02,719 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-14T06:19:02,727 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2a5ffc6{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-14T06:19:02,727 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7f30a77e{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T06:19:02,728 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T06:19:02,728 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@40029c9a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T06:19:02,728 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3f9fd429{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5ae4a4a-2751-c4f4-7a28-da4c7cf058aa/hadoop.log.dir/,STOPPED} 2024-11-14T06:19:02,744 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-14T06:19:02,918 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down